fix: stable and consistent paging of DNS records (#2)

- list DNS records with order=name&direction=asc, without an order
  Cloudflare's pages can overlap and miss records
- de-duplicate fetched entries by ID, fetch the pages again (up to 3
  times) if fewer entries than total_count were fetched, otherwise log
  a warning; debug log per fetched page
- testPaging: unique record names for an unambiguous order
- testDns: wait for the unfiltered record list, which can lag behind
  directly after creating records

refs #2

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
thischwaandClaude Opus 5.5 committed 2026-11-04 14:43:22 +01:00
1 parent 6eb6c2b778
commit 09eabb0d59
4 files changed
+160 -28

No files matched your search

+1
View File
@@ -33,6 +33,7 @@
- fixed: `zoneGet` threw an `IndexOutOfBoundsException` for unknown zones, it now returns `null` or throws a `CloudflareNotFoundException` (depending on `emptyResultThrowsException`)
- fixed: `getSubdomain()` failed for apex records and records with multi-level subdomains, batch operations now send the full record name
- fixed: `zoneList()` and `recordList(...)` without `PagingRequest` returned only the first page, now all pages are fetched
- fixed: DNS records are listed with a stable order (`order=name`), because without it Cloudflare's pages can overlap and miss records; fetched records are de-duplicated and inconsistent pages are fetched again
- fixed: `recordDelete` threw a `NullPointerException` for an empty result
- 0.5.0:
- moved the project to git.mein-gateway.de
@@ -97,6 +97,17 @@ public class CfDnsClient extends CfBasicHttpClient {
*/
static final int RECORDS_PER_PAGE = 1000;
/**
* Sort order for listing DNS records. Cloudflare doesn't guarantee a stable order without it,
* so pages could overlap or miss records.
*/
static final String RECORD_ORDER = "order=name&direction=asc";
/**
* Max. attempts to fetch all pages consistently.
*/
private static final int MAX_PAGING_ATTEMPTS = 3;
/**
* Conventional name for the zone apex (the zone name itself).
*/
@@ -285,7 +296,7 @@ public class CfDnsClient extends CfBasicHttpClient {
*/
public List<RecordEntity> recordList(ZoneEntity zone, @Nullable PagingRequest pagingRequest)
throws CloudflareApiException {
String path = CfRequest.RECORD_LIST.buildPath(zone.getId());
String path = appendQuery(CfRequest.RECORD_LIST.buildPath(zone.getId()), RECORD_ORDER);
if (pagingRequest == null) {
return filterAndSetZoneRecords(zone, null,
getAllPages(path, RecordMultipleResponse.class, recordsPerPage));
@@ -314,7 +325,8 @@ public class CfDnsClient extends CfBasicHttpClient {
@Nullable RecordType... types)
throws CloudflareApiException {
String fqdn = buildFqdn(zone, subdomain);
String endpoint = addTypeFilter(CfRequest.RECORD_LIST_NAME.buildPath(zone.getId(), fqdn), types);
String endpoint = appendQuery(
addTypeFilter(CfRequest.RECORD_LIST_NAME.buildPath(zone.getId(), fqdn), types), RECORD_ORDER);
List<RecordEntity> recs = getAllPages(endpoint, RecordMultipleResponse.class, recordsPerPage);
return filterAndSetZoneRecords(zone, types, recs);
}
@@ -332,7 +344,8 @@ public class CfDnsClient extends CfBasicHttpClient {
*/
public List<RecordEntity> recordListByType(ZoneEntity zone, @Nullable RecordType... types)
throws CloudflareApiException {
String endpoint = addTypeFilter(CfRequest.RECORD_LIST.buildPath(zone.getId()), types);
String endpoint = appendQuery(
addTypeFilter(CfRequest.RECORD_LIST.buildPath(zone.getId()), types), RECORD_ORDER);
List<RecordEntity> recs = getAllPages(endpoint, RecordMultipleResponse.class, recordsPerPage);
return filterAndSetZoneRecords(zone, types, recs);
}
@@ -615,7 +628,11 @@ public class CfDnsClient extends CfBasicHttpClient {
if (types == null || types.length != 1 || types[0] == null) {
return endpoint;
}
return endpoint + (endpoint.contains("?") ? "&" : "?") + "type=" + types[0].getType();
return appendQuery(endpoint, "type=" + types[0].getType());
}
private static String appendQuery(String endpoint, String query) {
return endpoint + (endpoint.contains("?") ? "&" : "?") + query;
}
private void setZoneForBatchResults(@Nullable BatchEntry result, ZoneEntity zone) {
@@ -640,25 +657,64 @@ public class CfDnsClient extends CfBasicHttpClient {
/**
* Fetches all pages of a paginated endpoint and returns the combined results.
*
* <p>Entries are de-duplicated by their ID. If fewer entries than announced by the API were
* fetched (e.g. because of concurrent changes), the pages are fetched again, up to
* {@value #MAX_PAGING_ATTEMPTS} times.
*
* @param endpoint the endpoint path, may already contain a query string
* @param responseType the expected response type
* @param perPage the page size to request
*/
private <E extends ResponseEntity, R extends AbstractMultipleResponse<E>> List<E> getAllPages(
String endpoint, Class<R> responseType, int perPage) throws CloudflareApiException {
List<E> all = new ArrayList<>();
List<E> result = List.of();
for (int attempt = 1; attempt <= MAX_PAGING_ATTEMPTS; attempt++) {
PagingResult<E> paged = fetchAllPagesOnce(endpoint, responseType, perPage);
result = paged.entries();
if (paged.expectedCount() < 0 || result.size() >= paged.expectedCount()) {
return result;
}
log.debug("Fetched {} of {} entries from {} (attempt {}), fetching again.", result.size(),
paged.expectedCount(), endpoint, attempt);
}
log.warn("Paging of {} was inconsistent, the result may be incomplete.", endpoint);
return result;
}
private record PagingResult<E>(List<E> entries, int expectedCount) {
}
private <E extends ResponseEntity, R extends AbstractMultipleResponse<E>> PagingResult<E> fetchAllPagesOnce(
String endpoint, Class<R> responseType, int perPage) throws CloudflareApiException {
Map<String, E> byId = new LinkedHashMap<>();
List<E> withoutId = new ArrayList<>();
int expectedCount = -1;
int page = 1;
while (true) {
String pagedEndpoint = PagingRequest.of(page, perPage).addQueryString(endpoint);
R resp = getRequest(pagedEndpoint, responseType);
checkResponse(resp);
List<E> result = resp.getResult();
if (result != null) {
all.addAll(result);
List<E> pageResult = resp.getResult();
if (pageResult != null) {
for (E entry : pageResult) {
if (entry.getId() == null) {
withoutId.add(entry);
} else {
byId.putIfAbsent(entry.getId(), entry);
}
}
}
ResultInfo info = resp.getResultInfo();
if (info == null || result == null || result.isEmpty() || page >= info.totalPages()) {
return all;
if (info != null && page == 1) {
expectedCount = info.totalCount();
}
log.debug("Fetched page {} of {} from {}: {} entries, total {}", page,
info == null ? "?" : info.totalPages(), endpoint,
pageResult == null ? 0 : pageResult.size(), info == null ? "?" : info.totalCount());
if (info == null || pageResult == null || pageResult.isEmpty() || page >= info.totalPages()) {
List<E> all = new ArrayList<>(byId.values());
all.addAll(withoutId);
return new PagingResult<>(all, info == null || page == 1 ? -1 : expectedCount);
}
page++;
}
@@ -194,8 +194,9 @@ public class CfClientTest {
}
}
// test recordList without SLD
List<RecordEntity> fullList = client.recordList(z);
// test recordList without SLD (the unfiltered list may lag behind directly after creating records)
List<RecordEntity> fullList = awaitRecords(() -> client.recordList(z),
recs -> recs.stream().anyMatch(re -> re.getId().equals(createdRe2.getId())));
assertTrue(fullList.size() >= 2);
assertTrue(fullList.stream().anyMatch(re -> re.getId().equals(createdRe1.getId())));
assertTrue(fullList.stream().anyMatch(re -> re.getId().equals(createdRe2.getId())));
@@ -203,7 +204,8 @@ public class CfClientTest {
fullList.stream().allMatch(re -> re.getType().equals(RecordType.A.getType()) || re.getType().equals(RecordType.AAAA.getType())));
// test recordList with types without SLD
List<RecordEntity> aList = client.recordListByType(z, RecordType.A);
List<RecordEntity> aList = awaitRecords(() -> client.recordListByType(z, RecordType.A),
recs -> recs.stream().anyMatch(re -> re.getId().equals(createdRe1.getId())));
assertFalse(aList.isEmpty());
assertTrue(aList.stream().anyMatch(re -> re.getId().equals(createdRe1.getId())));
assertTrue(aList.stream().noneMatch(re -> re.getId().equals(createdRe2.getId())));
@@ -473,6 +475,7 @@ public class CfClientTest {
void testPaging() throws Exception {
ZoneEntity zone = client.zoneGet(ZONE_STR);
String pagingSld = "paging-" + System.currentTimeMillis();
List<RecordEntity> createdRecords = new ArrayList<>();
try {
int existingCount = 0;
@@ -488,10 +491,11 @@ public class CfClientTest {
int targetCount = 12;
int recordsToCreate = Math.max(0, targetCount - existingCount);
// Create additional A records if needed
List<RecordEntity> createdRecords = new ArrayList<>();
// Create additional A records if needed. Each record gets its own name, so the order by name
// is unambiguous (Cloudflare doesn't guarantee a stable order of records with the same name).
for (int i = 1; i <= recordsToCreate; i++) {
RecordEntity record = RecordEntity.build(pagingSld, RecordType.A, TTL, "127.0.0." + i);
String name = String.format("%s-%02d.%s", pagingSld, i, ZONE_STR);
RecordEntity record = RecordEntity.build(name, RecordType.A, TTL, "127.0.0." + i);
RecordEntity created = client.recordCreate(zone, record);
createdRecords.add(created);
assertNotNull(created.getId());
@@ -532,13 +536,11 @@ public class CfClientTest {
new CfDnsClientBuilder.ApiTokenAuth(API_TOKEN), 5)) {
assertEquals(allRecordIds, smallPageClient.recordList(zone).stream()
.map(RecordEntity::getId).collect(Collectors.toSet()));
assertEquals(createdRecords.size(), smallPageClient.recordListBySubdomain(zone, pagingSld).size());
assertEquals(createdRecords.size(), smallPageClient.recordListByType(zone, RecordType.A).size());
assertEquals(1, smallPageClient.recordListBySubdomain(zone, pagingSld + "-01").size());
}
} finally {
try {
client.recordDeleteTypeIfExists(zone, pagingSld, RecordType.A);
} catch (Exception e) { /* ignore */ }
deleteQuietly(zone, createdRecords);
}
}
@@ -724,4 +726,32 @@ public class CfClientTest {
}
}
}
@FunctionalInterface
private interface RecordQuery {
List<RecordEntity> get() throws CloudflareApiException;
}
/**
* Repeats the query until the condition is met. The unfiltered record list of Cloudflare can lag
* behind for a short time after records were created.
*/
private static List<RecordEntity> awaitRecords(RecordQuery query,
java.util.function.Predicate<List<RecordEntity>> condition)
throws CloudflareApiException, InterruptedException {
List<RecordEntity> recs = List.of();
for (int attempt = 1; attempt <= 10; attempt++) {
try {
recs = query.get();
if (condition.test(recs)) {
return recs;
}
} catch (CloudflareNotFoundException e) {
recs = List.of();
}
log.info("Records not yet listed (attempt {}), waiting ...", attempt);
Thread.sleep(1000);
}
return recs;
}
}
@@ -471,7 +471,7 @@ class CfDnsClientMockTest {
@Test
void testRecordListByType_FetchesAllPages() throws Exception {
ZoneEntity zone = createTestZone();
String base = "/zones/" + TEST_ZONE_ID + "/dns_records?type=A&page=%d&per_page=" + CfDnsClient.RECORDS_PER_PAGE;
String base = "/zones/" + TEST_ZONE_ID + "/dns_records?type=A&order=name&direction=asc&page=%d&per_page=" + CfDnsClient.RECORDS_PER_PAGE;
CfDnsClient spyClient = spy(client);
doReturn(recordResponse(List.of(RecordEntity.build("a.example.com", RecordType.A, 60, "1.1.1.1")),
new ResultInfo(1, 1, 2, 1, 2)))
@@ -489,7 +489,7 @@ class CfDnsClientMockTest {
@Test
void testRecordListBySld_FetchesAllPages() throws Exception {
ZoneEntity zone = createTestZone();
String base = "/zones/" + TEST_ZONE_ID + "/dns_records?name=test.example.com&page=%d&per_page="
String base = "/zones/" + TEST_ZONE_ID + "/dns_records?name=test.example.com&order=name&direction=asc&page=%d&per_page="
+ CfDnsClient.RECORDS_PER_PAGE;
CfDnsClient spyClient = spy(client);
doReturn(recordResponse(List.of(RecordEntity.build("test.example.com", RecordType.A, 60, "1.1.1.1")),
@@ -529,7 +529,7 @@ class CfDnsClientMockTest {
spyClient.recordListBySubdomain(zone, CfDnsClient.APEX);
verify(spyClient).getRequest(
eq("/zones/" + TEST_ZONE_ID + "/dns_records?name=example.com&page=1&per_page="
eq("/zones/" + TEST_ZONE_ID + "/dns_records?name=example.com&order=name&direction=asc&page=1&per_page="
+ CfDnsClient.RECORDS_PER_PAGE),
eq(RecordMultipleResponse.class));
}
@@ -600,10 +600,10 @@ class CfDnsClientMockTest {
spyClient.recordListByType(createTestZone(), RecordType.MX);
spyClient.recordListBySubdomain(createTestZone(), "www", RecordType.TXT);
verify(spyClient).getRequest(eq("/zones/" + TEST_ZONE_ID + "/dns_records?type=MX&page=1&per_page="
verify(spyClient).getRequest(eq("/zones/" + TEST_ZONE_ID + "/dns_records?type=MX&order=name&direction=asc&page=1&per_page="
+ CfDnsClient.RECORDS_PER_PAGE), eq(RecordMultipleResponse.class));
verify(spyClient).getRequest(eq("/zones/" + TEST_ZONE_ID
+ "/dns_records?name=www.example.com&type=TXT&page=1&per_page="
+ "/dns_records?name=www.example.com&type=TXT&order=name&direction=asc&page=1&per_page="
+ CfDnsClient.RECORDS_PER_PAGE), eq(RecordMultipleResponse.class));
}
@@ -624,7 +624,7 @@ class CfDnsClientMockTest {
List<RecordEntity> records = spyClient.recordListByType(createTestZone(), RecordType.A, RecordType.AAAA);
assertEquals(List.of("A", "AAAA"), records.stream().map(RecordEntity::getType).toList());
verify(spyClient).getRequest(eq("/zones/" + TEST_ZONE_ID + "/dns_records?page=1&per_page="
verify(spyClient).getRequest(eq("/zones/" + TEST_ZONE_ID + "/dns_records?order=name&direction=asc&page=1&per_page="
+ CfDnsClient.RECORDS_PER_PAGE), eq(RecordMultipleResponse.class));
}
@@ -759,10 +759,10 @@ class CfDnsClientMockTest {
spyClient.recordListBySubdomain(createTestZone(), "0/26");
verify(spyClient).getRequest(eq("/zones/" + TEST_ZONE_ID
+ "/dns_records?name=%24%28reboot%29.example.com&page=1&per_page=" + CfDnsClient.RECORDS_PER_PAGE),
+ "/dns_records?name=%24%28reboot%29.example.com&order=name&direction=asc&page=1&per_page=" + CfDnsClient.RECORDS_PER_PAGE),
eq(RecordMultipleResponse.class));
verify(spyClient).getRequest(eq("/zones/" + TEST_ZONE_ID
+ "/dns_records?name=0%2F26.example.com&page=1&per_page=" + CfDnsClient.RECORDS_PER_PAGE),
+ "/dns_records?name=0%2F26.example.com&order=name&direction=asc&page=1&per_page=" + CfDnsClient.RECORDS_PER_PAGE),
eq(RecordMultipleResponse.class));
}
@@ -779,4 +779,49 @@ class CfDnsClientMockTest {
verify(spyClient, never()).patchRequest(anyString(), any(), any());
verify(spyClient, never()).getRequest(anyString(), any());
}
private static RecordEntity rec(String id, String name) {
RecordEntity rec = RecordEntity.build(name, RecordType.A, 60, "1.1.1.1");
rec.setId(id);
return rec;
}
private static String recordsPage(int page) {
return "/zones/" + TEST_ZONE_ID + "/dns_records?order=name&direction=asc&page=" + page
+ "&per_page=" + CfDnsClient.RECORDS_PER_PAGE;
}
@Test
void testRecordList_InconsistentPagesAreFetchedAgain() throws Exception {
CfDnsClient spyClient = spy(client);
// 1st attempt: page 2 repeats a record of page 1 (unstable order), so record c is missing
doReturn(recordResponse(List.of(rec("a", "a.example.com"), rec("b", "b.example.com")),
new ResultInfo(1, 2, 2, 2, 3)))
.when(spyClient).getRequest(eq(recordsPage(1)), eq(RecordMultipleResponse.class));
doReturn(recordResponse(List.of(rec("b", "b.example.com")), new ResultInfo(2, 2, 2, 1, 3)),
recordResponse(List.of(rec("c", "c.example.com")), new ResultInfo(2, 2, 2, 1, 3)))
.when(spyClient).getRequest(eq(recordsPage(2)), eq(RecordMultipleResponse.class));
List<RecordEntity> records = spyClient.recordList(createTestZone());
assertEquals(List.of("a", "b", "c"), records.stream().map(RecordEntity::getId).toList());
verify(spyClient, times(2)).getRequest(eq(recordsPage(1)), eq(RecordMultipleResponse.class));
verify(spyClient, times(2)).getRequest(eq(recordsPage(2)), eq(RecordMultipleResponse.class));
}
@Test
void testRecordList_PersistentlyInconsistentPagesReturnDeduplicatedResult() throws Exception {
CfDnsClient spyClient = spy(client);
doReturn(recordResponse(List.of(rec("a", "a.example.com"), rec("b", "b.example.com")),
new ResultInfo(1, 2, 2, 2, 3)))
.when(spyClient).getRequest(eq(recordsPage(1)), eq(RecordMultipleResponse.class));
doReturn(recordResponse(List.of(rec("b", "b.example.com")), new ResultInfo(2, 2, 2, 1, 3)))
.when(spyClient).getRequest(eq(recordsPage(2)), eq(RecordMultipleResponse.class));
List<RecordEntity> records = spyClient.recordList(createTestZone());
// no duplicates, after 3 attempts the incomplete result is returned
assertEquals(List.of("a", "b"), records.stream().map(RecordEntity::getId).toList());
verify(spyClient, times(3)).getRequest(eq(recordsPage(1)), eq(RecordMultipleResponse.class));
}
}