fix: stable and consistent paging of DNS records (#2)
- list DNS records with order=name&direction=asc, without an order Cloudflare's pages can overlap and miss records - de-duplicate fetched entries by ID, fetch the pages again (up to 3 times) if fewer entries than total_count were fetched, otherwise log a warning; debug log per fetched page - testPaging: unique record names for an unambiguous order - testDns: wait for the unfiltered record list, which can lag behind directly after creating records refs #2 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
6eb6c2b778
commit
09eabb0d59
4 files changed
+160
-28
No files matched your search
@@ -97,6 +97,17 @@ public class CfDnsClient extends CfBasicHttpClient {
|
||||
*/
|
||||
static final int RECORDS_PER_PAGE = 1000;
|
||||
|
||||
/**
|
||||
* Sort order for listing DNS records. Cloudflare doesn't guarantee a stable order without it,
|
||||
* so pages could overlap or miss records.
|
||||
*/
|
||||
static final String RECORD_ORDER = "order=name&direction=asc";
|
||||
|
||||
/**
|
||||
* Max. attempts to fetch all pages consistently.
|
||||
*/
|
||||
private static final int MAX_PAGING_ATTEMPTS = 3;
|
||||
|
||||
/**
|
||||
* Conventional name for the zone apex (the zone name itself).
|
||||
*/
|
||||
@@ -285,7 +296,7 @@ public class CfDnsClient extends CfBasicHttpClient {
|
||||
*/
|
||||
public List<RecordEntity> recordList(ZoneEntity zone, @Nullable PagingRequest pagingRequest)
|
||||
throws CloudflareApiException {
|
||||
String path = CfRequest.RECORD_LIST.buildPath(zone.getId());
|
||||
String path = appendQuery(CfRequest.RECORD_LIST.buildPath(zone.getId()), RECORD_ORDER);
|
||||
if (pagingRequest == null) {
|
||||
return filterAndSetZoneRecords(zone, null,
|
||||
getAllPages(path, RecordMultipleResponse.class, recordsPerPage));
|
||||
@@ -314,7 +325,8 @@ public class CfDnsClient extends CfBasicHttpClient {
|
||||
@Nullable RecordType... types)
|
||||
throws CloudflareApiException {
|
||||
String fqdn = buildFqdn(zone, subdomain);
|
||||
String endpoint = addTypeFilter(CfRequest.RECORD_LIST_NAME.buildPath(zone.getId(), fqdn), types);
|
||||
String endpoint = appendQuery(
|
||||
addTypeFilter(CfRequest.RECORD_LIST_NAME.buildPath(zone.getId(), fqdn), types), RECORD_ORDER);
|
||||
List<RecordEntity> recs = getAllPages(endpoint, RecordMultipleResponse.class, recordsPerPage);
|
||||
return filterAndSetZoneRecords(zone, types, recs);
|
||||
}
|
||||
@@ -332,7 +344,8 @@ public class CfDnsClient extends CfBasicHttpClient {
|
||||
*/
|
||||
public List<RecordEntity> recordListByType(ZoneEntity zone, @Nullable RecordType... types)
|
||||
throws CloudflareApiException {
|
||||
String endpoint = addTypeFilter(CfRequest.RECORD_LIST.buildPath(zone.getId()), types);
|
||||
String endpoint = appendQuery(
|
||||
addTypeFilter(CfRequest.RECORD_LIST.buildPath(zone.getId()), types), RECORD_ORDER);
|
||||
List<RecordEntity> recs = getAllPages(endpoint, RecordMultipleResponse.class, recordsPerPage);
|
||||
return filterAndSetZoneRecords(zone, types, recs);
|
||||
}
|
||||
@@ -615,7 +628,11 @@ public class CfDnsClient extends CfBasicHttpClient {
|
||||
if (types == null || types.length != 1 || types[0] == null) {
|
||||
return endpoint;
|
||||
}
|
||||
return endpoint + (endpoint.contains("?") ? "&" : "?") + "type=" + types[0].getType();
|
||||
return appendQuery(endpoint, "type=" + types[0].getType());
|
||||
}
|
||||
|
||||
private static String appendQuery(String endpoint, String query) {
|
||||
return endpoint + (endpoint.contains("?") ? "&" : "?") + query;
|
||||
}
|
||||
|
||||
private void setZoneForBatchResults(@Nullable BatchEntry result, ZoneEntity zone) {
|
||||
@@ -640,25 +657,64 @@ public class CfDnsClient extends CfBasicHttpClient {
|
||||
/**
|
||||
* Fetches all pages of a paginated endpoint and returns the combined results.
|
||||
*
|
||||
* <p>Entries are de-duplicated by their ID. If fewer entries than announced by the API were
|
||||
* fetched (e.g. because of concurrent changes), the pages are fetched again, up to
|
||||
* {@value #MAX_PAGING_ATTEMPTS} times.
|
||||
*
|
||||
* @param endpoint the endpoint path, may already contain a query string
|
||||
* @param responseType the expected response type
|
||||
* @param perPage the page size to request
|
||||
*/
|
||||
private <E extends ResponseEntity, R extends AbstractMultipleResponse<E>> List<E> getAllPages(
|
||||
String endpoint, Class<R> responseType, int perPage) throws CloudflareApiException {
|
||||
List<E> all = new ArrayList<>();
|
||||
List<E> result = List.of();
|
||||
for (int attempt = 1; attempt <= MAX_PAGING_ATTEMPTS; attempt++) {
|
||||
PagingResult<E> paged = fetchAllPagesOnce(endpoint, responseType, perPage);
|
||||
result = paged.entries();
|
||||
if (paged.expectedCount() < 0 || result.size() >= paged.expectedCount()) {
|
||||
return result;
|
||||
}
|
||||
log.debug("Fetched {} of {} entries from {} (attempt {}), fetching again.", result.size(),
|
||||
paged.expectedCount(), endpoint, attempt);
|
||||
}
|
||||
log.warn("Paging of {} was inconsistent, the result may be incomplete.", endpoint);
|
||||
return result;
|
||||
}
|
||||
|
||||
private record PagingResult<E>(List<E> entries, int expectedCount) {
|
||||
}
|
||||
|
||||
private <E extends ResponseEntity, R extends AbstractMultipleResponse<E>> PagingResult<E> fetchAllPagesOnce(
|
||||
String endpoint, Class<R> responseType, int perPage) throws CloudflareApiException {
|
||||
Map<String, E> byId = new LinkedHashMap<>();
|
||||
List<E> withoutId = new ArrayList<>();
|
||||
int expectedCount = -1;
|
||||
int page = 1;
|
||||
while (true) {
|
||||
String pagedEndpoint = PagingRequest.of(page, perPage).addQueryString(endpoint);
|
||||
R resp = getRequest(pagedEndpoint, responseType);
|
||||
checkResponse(resp);
|
||||
List<E> result = resp.getResult();
|
||||
if (result != null) {
|
||||
all.addAll(result);
|
||||
List<E> pageResult = resp.getResult();
|
||||
if (pageResult != null) {
|
||||
for (E entry : pageResult) {
|
||||
if (entry.getId() == null) {
|
||||
withoutId.add(entry);
|
||||
} else {
|
||||
byId.putIfAbsent(entry.getId(), entry);
|
||||
}
|
||||
}
|
||||
}
|
||||
ResultInfo info = resp.getResultInfo();
|
||||
if (info == null || result == null || result.isEmpty() || page >= info.totalPages()) {
|
||||
return all;
|
||||
if (info != null && page == 1) {
|
||||
expectedCount = info.totalCount();
|
||||
}
|
||||
log.debug("Fetched page {} of {} from {}: {} entries, total {}", page,
|
||||
info == null ? "?" : info.totalPages(), endpoint,
|
||||
pageResult == null ? 0 : pageResult.size(), info == null ? "?" : info.totalCount());
|
||||
if (info == null || pageResult == null || pageResult.isEmpty() || page >= info.totalPages()) {
|
||||
List<E> all = new ArrayList<>(byId.values());
|
||||
all.addAll(withoutId);
|
||||
return new PagingResult<>(all, info == null || page == 1 ? -1 : expectedCount);
|
||||
}
|
||||
page++;
|
||||
}
|
||||
|
||||
@@ -194,8 +194,9 @@ public class CfClientTest {
|
||||
}
|
||||
}
|
||||
|
||||
// test recordList without SLD
|
||||
List<RecordEntity> fullList = client.recordList(z);
|
||||
// test recordList without SLD (the unfiltered list may lag behind directly after creating records)
|
||||
List<RecordEntity> fullList = awaitRecords(() -> client.recordList(z),
|
||||
recs -> recs.stream().anyMatch(re -> re.getId().equals(createdRe2.getId())));
|
||||
assertTrue(fullList.size() >= 2);
|
||||
assertTrue(fullList.stream().anyMatch(re -> re.getId().equals(createdRe1.getId())));
|
||||
assertTrue(fullList.stream().anyMatch(re -> re.getId().equals(createdRe2.getId())));
|
||||
@@ -203,7 +204,8 @@ public class CfClientTest {
|
||||
fullList.stream().allMatch(re -> re.getType().equals(RecordType.A.getType()) || re.getType().equals(RecordType.AAAA.getType())));
|
||||
|
||||
// test recordList with types without SLD
|
||||
List<RecordEntity> aList = client.recordListByType(z, RecordType.A);
|
||||
List<RecordEntity> aList = awaitRecords(() -> client.recordListByType(z, RecordType.A),
|
||||
recs -> recs.stream().anyMatch(re -> re.getId().equals(createdRe1.getId())));
|
||||
assertFalse(aList.isEmpty());
|
||||
assertTrue(aList.stream().anyMatch(re -> re.getId().equals(createdRe1.getId())));
|
||||
assertTrue(aList.stream().noneMatch(re -> re.getId().equals(createdRe2.getId())));
|
||||
@@ -473,6 +475,7 @@ public class CfClientTest {
|
||||
void testPaging() throws Exception {
|
||||
ZoneEntity zone = client.zoneGet(ZONE_STR);
|
||||
String pagingSld = "paging-" + System.currentTimeMillis();
|
||||
List<RecordEntity> createdRecords = new ArrayList<>();
|
||||
|
||||
try {
|
||||
int existingCount = 0;
|
||||
@@ -488,10 +491,11 @@ public class CfClientTest {
|
||||
int targetCount = 12;
|
||||
int recordsToCreate = Math.max(0, targetCount - existingCount);
|
||||
|
||||
// Create additional A records if needed
|
||||
List<RecordEntity> createdRecords = new ArrayList<>();
|
||||
// Create additional A records if needed. Each record gets its own name, so the order by name
|
||||
// is unambiguous (Cloudflare doesn't guarantee a stable order of records with the same name).
|
||||
for (int i = 1; i <= recordsToCreate; i++) {
|
||||
RecordEntity record = RecordEntity.build(pagingSld, RecordType.A, TTL, "127.0.0." + i);
|
||||
String name = String.format("%s-%02d.%s", pagingSld, i, ZONE_STR);
|
||||
RecordEntity record = RecordEntity.build(name, RecordType.A, TTL, "127.0.0." + i);
|
||||
RecordEntity created = client.recordCreate(zone, record);
|
||||
createdRecords.add(created);
|
||||
assertNotNull(created.getId());
|
||||
@@ -532,13 +536,11 @@ public class CfClientTest {
|
||||
new CfDnsClientBuilder.ApiTokenAuth(API_TOKEN), 5)) {
|
||||
assertEquals(allRecordIds, smallPageClient.recordList(zone).stream()
|
||||
.map(RecordEntity::getId).collect(Collectors.toSet()));
|
||||
assertEquals(createdRecords.size(), smallPageClient.recordListBySubdomain(zone, pagingSld).size());
|
||||
assertEquals(createdRecords.size(), smallPageClient.recordListByType(zone, RecordType.A).size());
|
||||
assertEquals(1, smallPageClient.recordListBySubdomain(zone, pagingSld + "-01").size());
|
||||
}
|
||||
} finally {
|
||||
try {
|
||||
client.recordDeleteTypeIfExists(zone, pagingSld, RecordType.A);
|
||||
} catch (Exception e) { /* ignore */ }
|
||||
deleteQuietly(zone, createdRecords);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -724,4 +726,32 @@ public class CfClientTest {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@FunctionalInterface
|
||||
private interface RecordQuery {
|
||||
List<RecordEntity> get() throws CloudflareApiException;
|
||||
}
|
||||
|
||||
/**
|
||||
* Repeats the query until the condition is met. The unfiltered record list of Cloudflare can lag
|
||||
* behind for a short time after records were created.
|
||||
*/
|
||||
private static List<RecordEntity> awaitRecords(RecordQuery query,
|
||||
java.util.function.Predicate<List<RecordEntity>> condition)
|
||||
throws CloudflareApiException, InterruptedException {
|
||||
List<RecordEntity> recs = List.of();
|
||||
for (int attempt = 1; attempt <= 10; attempt++) {
|
||||
try {
|
||||
recs = query.get();
|
||||
if (condition.test(recs)) {
|
||||
return recs;
|
||||
}
|
||||
} catch (CloudflareNotFoundException e) {
|
||||
recs = List.of();
|
||||
}
|
||||
log.info("Records not yet listed (attempt {}), waiting ...", attempt);
|
||||
Thread.sleep(1000);
|
||||
}
|
||||
return recs;
|
||||
}
|
||||
}
|
||||
@@ -471,7 +471,7 @@ class CfDnsClientMockTest {
|
||||
@Test
|
||||
void testRecordListByType_FetchesAllPages() throws Exception {
|
||||
ZoneEntity zone = createTestZone();
|
||||
String base = "/zones/" + TEST_ZONE_ID + "/dns_records?type=A&page=%d&per_page=" + CfDnsClient.RECORDS_PER_PAGE;
|
||||
String base = "/zones/" + TEST_ZONE_ID + "/dns_records?type=A&order=name&direction=asc&page=%d&per_page=" + CfDnsClient.RECORDS_PER_PAGE;
|
||||
CfDnsClient spyClient = spy(client);
|
||||
doReturn(recordResponse(List.of(RecordEntity.build("a.example.com", RecordType.A, 60, "1.1.1.1")),
|
||||
new ResultInfo(1, 1, 2, 1, 2)))
|
||||
@@ -489,7 +489,7 @@ class CfDnsClientMockTest {
|
||||
@Test
|
||||
void testRecordListBySld_FetchesAllPages() throws Exception {
|
||||
ZoneEntity zone = createTestZone();
|
||||
String base = "/zones/" + TEST_ZONE_ID + "/dns_records?name=test.example.com&page=%d&per_page="
|
||||
String base = "/zones/" + TEST_ZONE_ID + "/dns_records?name=test.example.com&order=name&direction=asc&page=%d&per_page="
|
||||
+ CfDnsClient.RECORDS_PER_PAGE;
|
||||
CfDnsClient spyClient = spy(client);
|
||||
doReturn(recordResponse(List.of(RecordEntity.build("test.example.com", RecordType.A, 60, "1.1.1.1")),
|
||||
@@ -529,7 +529,7 @@ class CfDnsClientMockTest {
|
||||
spyClient.recordListBySubdomain(zone, CfDnsClient.APEX);
|
||||
|
||||
verify(spyClient).getRequest(
|
||||
eq("/zones/" + TEST_ZONE_ID + "/dns_records?name=example.com&page=1&per_page="
|
||||
eq("/zones/" + TEST_ZONE_ID + "/dns_records?name=example.com&order=name&direction=asc&page=1&per_page="
|
||||
+ CfDnsClient.RECORDS_PER_PAGE),
|
||||
eq(RecordMultipleResponse.class));
|
||||
}
|
||||
@@ -600,10 +600,10 @@ class CfDnsClientMockTest {
|
||||
spyClient.recordListByType(createTestZone(), RecordType.MX);
|
||||
spyClient.recordListBySubdomain(createTestZone(), "www", RecordType.TXT);
|
||||
|
||||
verify(spyClient).getRequest(eq("/zones/" + TEST_ZONE_ID + "/dns_records?type=MX&page=1&per_page="
|
||||
verify(spyClient).getRequest(eq("/zones/" + TEST_ZONE_ID + "/dns_records?type=MX&order=name&direction=asc&page=1&per_page="
|
||||
+ CfDnsClient.RECORDS_PER_PAGE), eq(RecordMultipleResponse.class));
|
||||
verify(spyClient).getRequest(eq("/zones/" + TEST_ZONE_ID
|
||||
+ "/dns_records?name=www.example.com&type=TXT&page=1&per_page="
|
||||
+ "/dns_records?name=www.example.com&type=TXT&order=name&direction=asc&page=1&per_page="
|
||||
+ CfDnsClient.RECORDS_PER_PAGE), eq(RecordMultipleResponse.class));
|
||||
}
|
||||
|
||||
@@ -624,7 +624,7 @@ class CfDnsClientMockTest {
|
||||
List<RecordEntity> records = spyClient.recordListByType(createTestZone(), RecordType.A, RecordType.AAAA);
|
||||
|
||||
assertEquals(List.of("A", "AAAA"), records.stream().map(RecordEntity::getType).toList());
|
||||
verify(spyClient).getRequest(eq("/zones/" + TEST_ZONE_ID + "/dns_records?page=1&per_page="
|
||||
verify(spyClient).getRequest(eq("/zones/" + TEST_ZONE_ID + "/dns_records?order=name&direction=asc&page=1&per_page="
|
||||
+ CfDnsClient.RECORDS_PER_PAGE), eq(RecordMultipleResponse.class));
|
||||
}
|
||||
|
||||
@@ -759,10 +759,10 @@ class CfDnsClientMockTest {
|
||||
spyClient.recordListBySubdomain(createTestZone(), "0/26");
|
||||
|
||||
verify(spyClient).getRequest(eq("/zones/" + TEST_ZONE_ID
|
||||
+ "/dns_records?name=%24%28reboot%29.example.com&page=1&per_page=" + CfDnsClient.RECORDS_PER_PAGE),
|
||||
+ "/dns_records?name=%24%28reboot%29.example.com&order=name&direction=asc&page=1&per_page=" + CfDnsClient.RECORDS_PER_PAGE),
|
||||
eq(RecordMultipleResponse.class));
|
||||
verify(spyClient).getRequest(eq("/zones/" + TEST_ZONE_ID
|
||||
+ "/dns_records?name=0%2F26.example.com&page=1&per_page=" + CfDnsClient.RECORDS_PER_PAGE),
|
||||
+ "/dns_records?name=0%2F26.example.com&order=name&direction=asc&page=1&per_page=" + CfDnsClient.RECORDS_PER_PAGE),
|
||||
eq(RecordMultipleResponse.class));
|
||||
}
|
||||
|
||||
@@ -779,4 +779,49 @@ class CfDnsClientMockTest {
|
||||
verify(spyClient, never()).patchRequest(anyString(), any(), any());
|
||||
verify(spyClient, never()).getRequest(anyString(), any());
|
||||
}
|
||||
|
||||
private static RecordEntity rec(String id, String name) {
|
||||
RecordEntity rec = RecordEntity.build(name, RecordType.A, 60, "1.1.1.1");
|
||||
rec.setId(id);
|
||||
return rec;
|
||||
}
|
||||
|
||||
private static String recordsPage(int page) {
|
||||
return "/zones/" + TEST_ZONE_ID + "/dns_records?order=name&direction=asc&page=" + page
|
||||
+ "&per_page=" + CfDnsClient.RECORDS_PER_PAGE;
|
||||
}
|
||||
|
||||
@Test
|
||||
void testRecordList_InconsistentPagesAreFetchedAgain() throws Exception {
|
||||
CfDnsClient spyClient = spy(client);
|
||||
// 1st attempt: page 2 repeats a record of page 1 (unstable order), so record c is missing
|
||||
doReturn(recordResponse(List.of(rec("a", "a.example.com"), rec("b", "b.example.com")),
|
||||
new ResultInfo(1, 2, 2, 2, 3)))
|
||||
.when(spyClient).getRequest(eq(recordsPage(1)), eq(RecordMultipleResponse.class));
|
||||
doReturn(recordResponse(List.of(rec("b", "b.example.com")), new ResultInfo(2, 2, 2, 1, 3)),
|
||||
recordResponse(List.of(rec("c", "c.example.com")), new ResultInfo(2, 2, 2, 1, 3)))
|
||||
.when(spyClient).getRequest(eq(recordsPage(2)), eq(RecordMultipleResponse.class));
|
||||
|
||||
List<RecordEntity> records = spyClient.recordList(createTestZone());
|
||||
|
||||
assertEquals(List.of("a", "b", "c"), records.stream().map(RecordEntity::getId).toList());
|
||||
verify(spyClient, times(2)).getRequest(eq(recordsPage(1)), eq(RecordMultipleResponse.class));
|
||||
verify(spyClient, times(2)).getRequest(eq(recordsPage(2)), eq(RecordMultipleResponse.class));
|
||||
}
|
||||
|
||||
@Test
|
||||
void testRecordList_PersistentlyInconsistentPagesReturnDeduplicatedResult() throws Exception {
|
||||
CfDnsClient spyClient = spy(client);
|
||||
doReturn(recordResponse(List.of(rec("a", "a.example.com"), rec("b", "b.example.com")),
|
||||
new ResultInfo(1, 2, 2, 2, 3)))
|
||||
.when(spyClient).getRequest(eq(recordsPage(1)), eq(RecordMultipleResponse.class));
|
||||
doReturn(recordResponse(List.of(rec("b", "b.example.com")), new ResultInfo(2, 2, 2, 1, 3)))
|
||||
.when(spyClient).getRequest(eq(recordsPage(2)), eq(RecordMultipleResponse.class));
|
||||
|
||||
List<RecordEntity> records = spyClient.recordList(createTestZone());
|
||||
|
||||
// no duplicates, after 3 attempts the incomplete result is returned
|
||||
assertEquals(List.of("a", "b"), records.stream().map(RecordEntity::getId).toList());
|
||||
verify(spyClient, times(3)).getRequest(eq(recordsPage(1)), eq(RecordMultipleResponse.class));
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user