mirror of
https://github.com/keycloak/keycloak.git
synced 2026-05-26 13:50:48 +00:00
pre-compute password denylist Bloom filter to speed up server startup
Fixes #47356 Signed-off-by: Faseela K <faseela.k@est.tech>
This commit is contained in:
Faseela K
GitHub
+76
-2
@@ -17,15 +17,19 @@
|
||||
|
||||
package org.keycloak.policy;
|
||||
|
||||
import java.io.BufferedInputStream;
|
||||
import java.io.BufferedOutputStream;
|
||||
import java.io.BufferedReader;
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.io.OutputStream;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.Paths;
|
||||
import java.text.DecimalFormat;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Objects;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.concurrent.ConcurrentMap;
|
||||
@@ -247,6 +251,32 @@ public class BlacklistPasswordPolicyProviderFactory implements PasswordPolicyPro
|
||||
return builder.build();
|
||||
}
|
||||
|
||||
/**
|
||||
* Builds a pre-computed Bloom filter (.bloom) file from a plaintext password denylist file.
|
||||
* Each line is treated as one password (lowercased before insertion).
|
||||
*
|
||||
* @param inputFile path to the plaintext password list (one password per line, UTF-8)
|
||||
* @param outputFile path for the generated .bloom file
|
||||
* @param fpp desired false-positive probability (e.g. 0.0001)
|
||||
* @throws IOException if the input file cannot be read or the output file cannot be written
|
||||
*/
|
||||
public static void buildBloomFile(Path inputFile, Path outputFile, double fpp) throws IOException {
|
||||
long count;
|
||||
try (var lines = Files.lines(inputFile, StandardCharsets.UTF_8)) {
|
||||
count = lines.count();
|
||||
}
|
||||
BloomFilter<String> filter = BloomFilter.create(
|
||||
Funnels.stringFunnel(StandardCharsets.UTF_8), Math.max(count, 1), fpp);
|
||||
try (var lines = Files.lines(inputFile, StandardCharsets.UTF_8)) {
|
||||
lines.map(s -> s.toLowerCase(Locale.ROOT)).forEach(filter::put);
|
||||
}
|
||||
try (OutputStream out = new BufferedOutputStream(Files.newOutputStream(outputFile))) {
|
||||
filter.writeTo(out);
|
||||
}
|
||||
LOG.infof("Built pre-computed denylist: input=%s passwords=%d fpp=%f output=%s",
|
||||
inputFile, count, fpp, outputFile);
|
||||
}
|
||||
|
||||
/**
|
||||
* A {@link PasswordBlacklist} describes a list of too easy to guess
|
||||
* or potentially leaked passwords that users should not be able to use.
|
||||
@@ -371,12 +401,56 @@ public class BlacklistPasswordPolicyProviderFactory implements PasswordPolicyPro
|
||||
}
|
||||
|
||||
/**
|
||||
* Loads the referenced blacklist into a {@link BloomFilter}.
|
||||
* Loads the denylist into a {@link BloomFilter}.
|
||||
* If the configured file ends with {@code .bloom}, it is loaded as a pre-computed Bloom filter binary.
|
||||
* Otherwise, it is read as a plaintext password list.
|
||||
*
|
||||
* @return the {@link BloomFilter} backing a password blacklist
|
||||
* @return the {@link BloomFilter} backing a password denylist
|
||||
*/
|
||||
private BloomFilter<String> load() {
|
||||
if (name.endsWith(".bloom")) {
|
||||
return loadFromBloom();
|
||||
}
|
||||
return loadFromPlaintext();
|
||||
}
|
||||
|
||||
/**
|
||||
* Fast path: deserialise a pre-computed Bloom filter binary (.bloom).
|
||||
* Emits a warning when the stored false-positive probability differs from the configured value.
|
||||
*
|
||||
* @return the deserialised {@link BloomFilter}
|
||||
* @throws IOException if the binary file cannot be read
|
||||
*/
|
||||
private BloomFilter<String> loadFromBloom() {
|
||||
try {
|
||||
LOG.infof("Loading pre-computed denylist start: name=%s path=%s", name, path);
|
||||
long loadStartMillis = System.currentTimeMillis();
|
||||
BloomFilter<String> filter;
|
||||
try (BufferedInputStream in = new BufferedInputStream(
|
||||
Files.newInputStream(path), BUFFER_SIZE_IN_BYTES)) {
|
||||
filter = BloomFilter.readFrom(in, Funnels.stringFunnel(StandardCharsets.UTF_8));
|
||||
}
|
||||
long loadTimeMillis = System.currentTimeMillis() - loadStartMillis;
|
||||
LOG.infof("Loading pre-computed denylist finished: name=%s path=%s expectedFpp=%s loadTime=%dms",
|
||||
name, path, filter.expectedFpp(), loadTimeMillis);
|
||||
if (Math.abs(filter.expectedFpp() - falsePositiveProbability) > 1e-9) {
|
||||
LOG.warnf("Pre-computed denylist '%s' has fpp=%.6f but configured fpp=%.6f. "
|
||||
+ "Regenerate the .bloom file with 'kc.sh tools build-password-denylist' if this is unintended.",
|
||||
name, filter.expectedFpp(), falsePositiveProbability);
|
||||
}
|
||||
return filter;
|
||||
} catch (IOException e) {
|
||||
throw new RuntimeException("Loading pre-computed denylist failed: path=" + path, e);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Slow path: build a BloomFilter from the plaintext denylist file.
|
||||
* Requires two passes: one to count passwords, one to insert them.
|
||||
*
|
||||
* @return a newly constructed {@link BloomFilter} populated from the plaintext file
|
||||
*/
|
||||
private BloomFilter<String> loadFromPlaintext() {
|
||||
try {
|
||||
LOG.infof("Loading blacklist start: name=%s path=%s", name, path);
|
||||
long loadStartMillis = System.currentTimeMillis();
|
||||
|
||||
+100
-19
@@ -1,5 +1,6 @@
|
||||
package org.keycloak.policy;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.Paths;
|
||||
@@ -18,34 +19,17 @@ public class BlacklistPasswordPolicyProviderTest {
|
||||
public TemporaryFolder tempFolder = new TemporaryFolder();
|
||||
|
||||
@Test
|
||||
public void testUpperCaseInFile() {
|
||||
public void testPasswordLookupIsCaseInsensitive() {
|
||||
FileBasedPasswordBlacklist blacklist =
|
||||
new FileBasedPasswordBlacklist(Paths.get("src/test/java/org/keycloak/policy"), "short_blacklist.txt",
|
||||
BlacklistPasswordPolicyProviderFactory.DEFAULT_FALSE_POSITIVE_PROBABILITY,
|
||||
BlacklistPasswordPolicyProviderFactory.DEFAULT_CHECK_INTERVAL_SECONDS * 1000L);
|
||||
|
||||
// all passwords in the deny list are in lower case
|
||||
// passwords in the deny list are stored in lower case; lookups must be case-insensitive
|
||||
Assert.assertFalse(blacklist.contains("1Password!"));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testAlwaysLowercaseInFile() {
|
||||
FileBasedPasswordBlacklist blacklist =
|
||||
new FileBasedPasswordBlacklist(Paths.get("src/test/java/org/keycloak/policy"), "short_blacklist.txt",
|
||||
BlacklistPasswordPolicyProviderFactory.DEFAULT_FALSE_POSITIVE_PROBABILITY,
|
||||
BlacklistPasswordPolicyProviderFactory.DEFAULT_CHECK_INTERVAL_SECONDS * 1000L);
|
||||
Assert.assertTrue(blacklist.contains("1Password!".toLowerCase()));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testLowerCaseInFile() {
|
||||
FileBasedPasswordBlacklist blacklist =
|
||||
new FileBasedPasswordBlacklist(Paths.get("src/test/java/org/keycloak/policy"), "short_blacklist.txt",
|
||||
BlacklistPasswordPolicyProviderFactory.DEFAULT_FALSE_POSITIVE_PROBABILITY,
|
||||
BlacklistPasswordPolicyProviderFactory.DEFAULT_CHECK_INTERVAL_SECONDS * 1000L);
|
||||
Assert.assertTrue(blacklist.contains("pass1!word"));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testReloadOnFileMtimeChange() throws Exception {
|
||||
Path file = tempFolder.newFile("blacklist.txt").toPath();
|
||||
@@ -111,4 +95,101 @@ public class BlacklistPasswordPolicyProviderTest {
|
||||
Assert.assertFalse(blacklist.contains("newpassword"));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testLoadFromBloomFile() throws Exception {
|
||||
// Write plaintext denylist and pre-compute .bloom alongside it
|
||||
Path txtFile = tempFolder.newFile("denylist.txt").toPath();
|
||||
Files.writeString(txtFile, "secret123\nbadpassword\n");
|
||||
writeBloomFile(tempFolder.getRoot().toPath(), "denylist.txt",
|
||||
BlacklistPasswordPolicyProviderFactory.DEFAULT_FALSE_POSITIVE_PROBABILITY);
|
||||
|
||||
// Admin explicitly configures the .bloom file
|
||||
FileBasedPasswordBlacklist denylist =
|
||||
new FileBasedPasswordBlacklist(tempFolder.getRoot().toPath(), "denylist.txt.bloom",
|
||||
BlacklistPasswordPolicyProviderFactory.DEFAULT_FALSE_POSITIVE_PROBABILITY, 0);
|
||||
|
||||
Assert.assertTrue("Should find password from .bloom file", denylist.contains("secret123"));
|
||||
Assert.assertTrue("Should find password from .bloom file", denylist.contains("badpassword"));
|
||||
Assert.assertFalse("Should not find password that was never added", denylist.contains("goodpassword"));
|
||||
}
|
||||
|
||||
@Test(expected = RuntimeException.class)
|
||||
public void testCorruptBloomFileThrows() throws Exception {
|
||||
// Write a corrupt (non-Guava) .bloom file
|
||||
Path bloomFile = tempFolder.newFile("denylist.txt.bloom").toPath();
|
||||
Files.writeString(bloomFile, "this is not a valid bloom filter binary");
|
||||
|
||||
// Admin configured .bloom explicitly — must fail hard on corrupt file, no fallback
|
||||
new FileBasedPasswordBlacklist(tempFolder.getRoot().toPath(), "denylist.txt.bloom",
|
||||
BlacklistPasswordPolicyProviderFactory.DEFAULT_FALSE_POSITIVE_PROBABILITY, 0);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testReloadBloomFileOnChange() throws Exception {
|
||||
Path txtFile = tempFolder.newFile("denylist.txt").toPath();
|
||||
Files.writeString(txtFile, "initialpassword\n");
|
||||
writeBloomFile(tempFolder.getRoot().toPath(), "denylist.txt",
|
||||
BlacklistPasswordPolicyProviderFactory.DEFAULT_FALSE_POSITIVE_PROBABILITY);
|
||||
|
||||
// Admin explicitly configures the .bloom file
|
||||
FileBasedPasswordBlacklist denylist =
|
||||
new FileBasedPasswordBlacklist(tempFolder.getRoot().toPath(), "denylist.txt.bloom",
|
||||
BlacklistPasswordPolicyProviderFactory.DEFAULT_FALSE_POSITIVE_PROBABILITY, 1);
|
||||
|
||||
Assert.assertTrue(denylist.contains("initialpassword"));
|
||||
Assert.assertFalse(denylist.contains("updatedpassword"));
|
||||
|
||||
// Update the plaintext file, regenerate and bump mtime on .bloom
|
||||
Thread.sleep(2);
|
||||
Files.writeString(txtFile, "updatedpassword\n");
|
||||
writeBloomFile(tempFolder.getRoot().toPath(), "denylist.txt",
|
||||
BlacklistPasswordPolicyProviderFactory.DEFAULT_FALSE_POSITIVE_PROBABILITY);
|
||||
Path bloomFile = txtFile.resolveSibling("denylist.txt.bloom");
|
||||
Files.setLastModifiedTime(bloomFile, FileTime.fromMillis(Files.getLastModifiedTime(bloomFile).toMillis() + 1000));
|
||||
|
||||
Assert.assertFalse("Old password should no longer be found after reload", denylist.contains("initialpassword"));
|
||||
Assert.assertTrue("Updated password should be found after reload", denylist.contains("updatedpassword"));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testBuildBloomFileCreatesReadableOutput() throws Exception {
|
||||
Path txtFile = tempFolder.newFile("denylist.txt").toPath();
|
||||
Files.writeString(txtFile, "alpha\nbeta\ngamma\n");
|
||||
|
||||
Path bloomFile = txtFile.resolveSibling("denylist.txt.bloom");
|
||||
BlacklistPasswordPolicyProviderFactory.buildBloomFile(
|
||||
txtFile, bloomFile, BlacklistPasswordPolicyProviderFactory.DEFAULT_FALSE_POSITIVE_PROBABILITY);
|
||||
Assert.assertTrue("bloom file must be created", Files.exists(bloomFile));
|
||||
Assert.assertTrue("bloom file must have non-zero size", Files.size(bloomFile) > 0);
|
||||
}
|
||||
|
||||
@Test(expected = IOException.class)
|
||||
public void testBuildBloomFileMissingInputThrows() throws Exception {
|
||||
Path missing = tempFolder.getRoot().toPath().resolve("nonexistent.txt");
|
||||
BlacklistPasswordPolicyProviderFactory.buildBloomFile(
|
||||
missing, missing.resolveSibling("nonexistent.txt.bloom"),
|
||||
BlacklistPasswordPolicyProviderFactory.DEFAULT_FALSE_POSITIVE_PROBABILITY);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testFppMismatchStillLoadsSuccessfully() throws Exception {
|
||||
Path txtFile = tempFolder.newFile("denylist.txt").toPath();
|
||||
Files.writeString(txtFile, "mismatchpw\n");
|
||||
|
||||
// Build bloom with a DIFFERENT fpp than what the server will use
|
||||
double bloomFpp = 0.01; // coarser
|
||||
double serverFpp = BlacklistPasswordPolicyProviderFactory.DEFAULT_FALSE_POSITIVE_PROBABILITY; // 0.0001
|
||||
BlacklistPasswordPolicyProviderFactory.buildBloomFile(txtFile, txtFile.resolveSibling("denylist.txt.bloom"), bloomFpp);
|
||||
|
||||
// Server should still load without throwing (only logs a warning)
|
||||
FileBasedPasswordBlacklist bl = new FileBasedPasswordBlacklist(
|
||||
tempFolder.getRoot().toPath(), "denylist.txt.bloom", serverFpp, 0);
|
||||
Assert.assertTrue("password must still be found despite fpp mismatch", bl.contains("mismatchpw"));
|
||||
}
|
||||
|
||||
private static void writeBloomFile(Path baseDir, String name, double fpp) throws IOException {
|
||||
Path input = baseDir.resolve(name);
|
||||
BlacklistPasswordPolicyProviderFactory.buildBloomFile(input, input.resolveSibling(name + ".bloom"), fpp);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user