From 09eabb0d59b1945ef7d2fd9418a20166661662dd Mon Sep 17 00:00:00 2001 From: Thilo Schwarz Date: Wed, 4 Nov 2026 14:43:22 +0100 Subject: [PATCH] fix: stable and consistent paging of DNS records (#2) - list DNS records with order=name&direction=asc, without an order Cloudflare's pages can overlap and miss records - de-duplicate fetched entries by ID, fetch the pages again (up to 3 times) if fewer entries than total_count were fetched, otherwise log a warning; debug log per fetched page - testPaging: unique record names for an unambiguous order - testDns: wait for the unfiltered record list, which can lag behind directly after creating records refs #2 Co-Authored-By: Claude Opus 5.5 --- changelog.md | 1 + .../java/codes/thischwa/cf/CfDnsClient.java | 76 ++++++++++++++++--- .../java/codes/thischwa/cf/CfClientTest.java | 50 +++++++++--- .../thischwa/cf/CfDnsClientMockTest.java | 61 +++++++++++++-- 4 files changed, 160 insertions(+), 28 deletions(-) diff --git a/changelog.md b/changelog.md index 59885bf..ad2007b 100644 --- a/changelog.md +++ b/changelog.md @@ -33,6 +33,7 @@ - fixed: `zoneGet` threw an `IndexOutOfBoundsException` for unknown zones, it now returns `null` or throws a `CloudflareNotFoundException` (depending on `emptyResultThrowsException`) - fixed: `getSubdomain()` failed for apex records and records with multi-level subdomains, batch operations now send the full record name - fixed: `zoneList()` and `recordList(...)` without `PagingRequest` returned only the first page, now all pages are fetched + - fixed: DNS records are listed with a stable order (`order=name`), because without it Cloudflare's pages can overlap and miss records; fetched records are de-duplicated and inconsistent pages are fetched again - fixed: `recordDelete` threw a `NullPointerException` for an empty result - 0.5.0: - moved the project to git.mein-gateway.de diff --git a/src/main/java/codes/thischwa/cf/CfDnsClient.java b/src/main/java/codes/thischwa/cf/CfDnsClient.java index a1f6d22..76633e0 100644 --- a/src/main/java/codes/thischwa/cf/CfDnsClient.java +++ b/src/main/java/codes/thischwa/cf/CfDnsClient.java @@ -97,6 +97,17 @@ public class CfDnsClient extends CfBasicHttpClient { */ static final int RECORDS_PER_PAGE = 1000; + /** + * Sort order for listing DNS records. Cloudflare doesn't guarantee a stable order without it, + * so pages could overlap or miss records. + */ + static final String RECORD_ORDER = "order=name&direction=asc"; + + /** + * Max. attempts to fetch all pages consistently. + */ + private static final int MAX_PAGING_ATTEMPTS = 3; + /** * Conventional name for the zone apex (the zone name itself). */ @@ -285,7 +296,7 @@ public class CfDnsClient extends CfBasicHttpClient { */ public List recordList(ZoneEntity zone, @Nullable PagingRequest pagingRequest) throws CloudflareApiException { - String path = CfRequest.RECORD_LIST.buildPath(zone.getId()); + String path = appendQuery(CfRequest.RECORD_LIST.buildPath(zone.getId()), RECORD_ORDER); if (pagingRequest == null) { return filterAndSetZoneRecords(zone, null, getAllPages(path, RecordMultipleResponse.class, recordsPerPage)); @@ -314,7 +325,8 @@ public class CfDnsClient extends CfBasicHttpClient { @Nullable RecordType... types) throws CloudflareApiException { String fqdn = buildFqdn(zone, subdomain); - String endpoint = addTypeFilter(CfRequest.RECORD_LIST_NAME.buildPath(zone.getId(), fqdn), types); + String endpoint = appendQuery( + addTypeFilter(CfRequest.RECORD_LIST_NAME.buildPath(zone.getId(), fqdn), types), RECORD_ORDER); List recs = getAllPages(endpoint, RecordMultipleResponse.class, recordsPerPage); return filterAndSetZoneRecords(zone, types, recs); } @@ -332,7 +344,8 @@ public class CfDnsClient extends CfBasicHttpClient { */ public List recordListByType(ZoneEntity zone, @Nullable RecordType... types) throws CloudflareApiException { - String endpoint = addTypeFilter(CfRequest.RECORD_LIST.buildPath(zone.getId()), types); + String endpoint = appendQuery( + addTypeFilter(CfRequest.RECORD_LIST.buildPath(zone.getId()), types), RECORD_ORDER); List recs = getAllPages(endpoint, RecordMultipleResponse.class, recordsPerPage); return filterAndSetZoneRecords(zone, types, recs); } @@ -615,7 +628,11 @@ public class CfDnsClient extends CfBasicHttpClient { if (types == null || types.length != 1 || types[0] == null) { return endpoint; } - return endpoint + (endpoint.contains("?") ? "&" : "?") + "type=" + types[0].getType(); + return appendQuery(endpoint, "type=" + types[0].getType()); + } + + private static String appendQuery(String endpoint, String query) { + return endpoint + (endpoint.contains("?") ? "&" : "?") + query; } private void setZoneForBatchResults(@Nullable BatchEntry result, ZoneEntity zone) { @@ -640,25 +657,64 @@ public class CfDnsClient extends CfBasicHttpClient { /** * Fetches all pages of a paginated endpoint and returns the combined results. * + *

Entries are de-duplicated by their ID. If fewer entries than announced by the API were + * fetched (e.g. because of concurrent changes), the pages are fetched again, up to + * {@value #MAX_PAGING_ATTEMPTS} times. + * * @param endpoint the endpoint path, may already contain a query string * @param responseType the expected response type * @param perPage the page size to request */ private > List getAllPages( String endpoint, Class responseType, int perPage) throws CloudflareApiException { - List all = new ArrayList<>(); + List result = List.of(); + for (int attempt = 1; attempt <= MAX_PAGING_ATTEMPTS; attempt++) { + PagingResult paged = fetchAllPagesOnce(endpoint, responseType, perPage); + result = paged.entries(); + if (paged.expectedCount() < 0 || result.size() >= paged.expectedCount()) { + return result; + } + log.debug("Fetched {} of {} entries from {} (attempt {}), fetching again.", result.size(), + paged.expectedCount(), endpoint, attempt); + } + log.warn("Paging of {} was inconsistent, the result may be incomplete.", endpoint); + return result; + } + + private record PagingResult(List entries, int expectedCount) { + } + + private > PagingResult fetchAllPagesOnce( + String endpoint, Class responseType, int perPage) throws CloudflareApiException { + Map byId = new LinkedHashMap<>(); + List withoutId = new ArrayList<>(); + int expectedCount = -1; int page = 1; while (true) { String pagedEndpoint = PagingRequest.of(page, perPage).addQueryString(endpoint); R resp = getRequest(pagedEndpoint, responseType); checkResponse(resp); - List result = resp.getResult(); - if (result != null) { - all.addAll(result); + List pageResult = resp.getResult(); + if (pageResult != null) { + for (E entry : pageResult) { + if (entry.getId() == null) { + withoutId.add(entry); + } else { + byId.putIfAbsent(entry.getId(), entry); + } + } } ResultInfo info = resp.getResultInfo(); - if (info == null || result == null || result.isEmpty() || page >= info.totalPages()) { - return all; + if (info != null && page == 1) { + expectedCount = info.totalCount(); + } + log.debug("Fetched page {} of {} from {}: {} entries, total {}", page, + info == null ? "?" : info.totalPages(), endpoint, + pageResult == null ? 0 : pageResult.size(), info == null ? "?" : info.totalCount()); + if (info == null || pageResult == null || pageResult.isEmpty() || page >= info.totalPages()) { + List all = new ArrayList<>(byId.values()); + all.addAll(withoutId); + return new PagingResult<>(all, info == null || page == 1 ? -1 : expectedCount); } page++; } diff --git a/src/test/java/codes/thischwa/cf/CfClientTest.java b/src/test/java/codes/thischwa/cf/CfClientTest.java index 635c4e1..f09bc6c 100644 --- a/src/test/java/codes/thischwa/cf/CfClientTest.java +++ b/src/test/java/codes/thischwa/cf/CfClientTest.java @@ -194,8 +194,9 @@ public class CfClientTest { } } - // test recordList without SLD - List fullList = client.recordList(z); + // test recordList without SLD (the unfiltered list may lag behind directly after creating records) + List fullList = awaitRecords(() -> client.recordList(z), + recs -> recs.stream().anyMatch(re -> re.getId().equals(createdRe2.getId()))); assertTrue(fullList.size() >= 2); assertTrue(fullList.stream().anyMatch(re -> re.getId().equals(createdRe1.getId()))); assertTrue(fullList.stream().anyMatch(re -> re.getId().equals(createdRe2.getId()))); @@ -203,7 +204,8 @@ public class CfClientTest { fullList.stream().allMatch(re -> re.getType().equals(RecordType.A.getType()) || re.getType().equals(RecordType.AAAA.getType()))); // test recordList with types without SLD - List aList = client.recordListByType(z, RecordType.A); + List aList = awaitRecords(() -> client.recordListByType(z, RecordType.A), + recs -> recs.stream().anyMatch(re -> re.getId().equals(createdRe1.getId()))); assertFalse(aList.isEmpty()); assertTrue(aList.stream().anyMatch(re -> re.getId().equals(createdRe1.getId()))); assertTrue(aList.stream().noneMatch(re -> re.getId().equals(createdRe2.getId()))); @@ -473,6 +475,7 @@ public class CfClientTest { void testPaging() throws Exception { ZoneEntity zone = client.zoneGet(ZONE_STR); String pagingSld = "paging-" + System.currentTimeMillis(); + List createdRecords = new ArrayList<>(); try { int existingCount = 0; @@ -488,10 +491,11 @@ public class CfClientTest { int targetCount = 12; int recordsToCreate = Math.max(0, targetCount - existingCount); - // Create additional A records if needed - List createdRecords = new ArrayList<>(); + // Create additional A records if needed. Each record gets its own name, so the order by name + // is unambiguous (Cloudflare doesn't guarantee a stable order of records with the same name). for (int i = 1; i <= recordsToCreate; i++) { - RecordEntity record = RecordEntity.build(pagingSld, RecordType.A, TTL, "127.0.0." + i); + String name = String.format("%s-%02d.%s", pagingSld, i, ZONE_STR); + RecordEntity record = RecordEntity.build(name, RecordType.A, TTL, "127.0.0." + i); RecordEntity created = client.recordCreate(zone, record); createdRecords.add(created); assertNotNull(created.getId()); @@ -532,13 +536,11 @@ public class CfClientTest { new CfDnsClientBuilder.ApiTokenAuth(API_TOKEN), 5)) { assertEquals(allRecordIds, smallPageClient.recordList(zone).stream() .map(RecordEntity::getId).collect(Collectors.toSet())); - assertEquals(createdRecords.size(), smallPageClient.recordListBySubdomain(zone, pagingSld).size()); assertEquals(createdRecords.size(), smallPageClient.recordListByType(zone, RecordType.A).size()); + assertEquals(1, smallPageClient.recordListBySubdomain(zone, pagingSld + "-01").size()); } } finally { - try { - client.recordDeleteTypeIfExists(zone, pagingSld, RecordType.A); - } catch (Exception e) { /* ignore */ } + deleteQuietly(zone, createdRecords); } } @@ -724,4 +726,32 @@ public class CfClientTest { } } } + + @FunctionalInterface + private interface RecordQuery { + List get() throws CloudflareApiException; + } + + /** + * Repeats the query until the condition is met. The unfiltered record list of Cloudflare can lag + * behind for a short time after records were created. + */ + private static List awaitRecords(RecordQuery query, + java.util.function.Predicate> condition) + throws CloudflareApiException, InterruptedException { + List recs = List.of(); + for (int attempt = 1; attempt <= 10; attempt++) { + try { + recs = query.get(); + if (condition.test(recs)) { + return recs; + } + } catch (CloudflareNotFoundException e) { + recs = List.of(); + } + log.info("Records not yet listed (attempt {}), waiting ...", attempt); + Thread.sleep(1000); + } + return recs; + } } diff --git a/src/test/java/codes/thischwa/cf/CfDnsClientMockTest.java b/src/test/java/codes/thischwa/cf/CfDnsClientMockTest.java index 3cf2473..bb0565c 100644 --- a/src/test/java/codes/thischwa/cf/CfDnsClientMockTest.java +++ b/src/test/java/codes/thischwa/cf/CfDnsClientMockTest.java @@ -471,7 +471,7 @@ class CfDnsClientMockTest { @Test void testRecordListByType_FetchesAllPages() throws Exception { ZoneEntity zone = createTestZone(); - String base = "/zones/" + TEST_ZONE_ID + "/dns_records?type=A&page=%d&per_page=" + CfDnsClient.RECORDS_PER_PAGE; + String base = "/zones/" + TEST_ZONE_ID + "/dns_records?type=A&order=name&direction=asc&page=%d&per_page=" + CfDnsClient.RECORDS_PER_PAGE; CfDnsClient spyClient = spy(client); doReturn(recordResponse(List.of(RecordEntity.build("a.example.com", RecordType.A, 60, "1.1.1.1")), new ResultInfo(1, 1, 2, 1, 2))) @@ -489,7 +489,7 @@ class CfDnsClientMockTest { @Test void testRecordListBySld_FetchesAllPages() throws Exception { ZoneEntity zone = createTestZone(); - String base = "/zones/" + TEST_ZONE_ID + "/dns_records?name=test.example.com&page=%d&per_page=" + String base = "/zones/" + TEST_ZONE_ID + "/dns_records?name=test.example.com&order=name&direction=asc&page=%d&per_page=" + CfDnsClient.RECORDS_PER_PAGE; CfDnsClient spyClient = spy(client); doReturn(recordResponse(List.of(RecordEntity.build("test.example.com", RecordType.A, 60, "1.1.1.1")), @@ -529,7 +529,7 @@ class CfDnsClientMockTest { spyClient.recordListBySubdomain(zone, CfDnsClient.APEX); verify(spyClient).getRequest( - eq("/zones/" + TEST_ZONE_ID + "/dns_records?name=example.com&page=1&per_page=" + eq("/zones/" + TEST_ZONE_ID + "/dns_records?name=example.com&order=name&direction=asc&page=1&per_page=" + CfDnsClient.RECORDS_PER_PAGE), eq(RecordMultipleResponse.class)); } @@ -600,10 +600,10 @@ class CfDnsClientMockTest { spyClient.recordListByType(createTestZone(), RecordType.MX); spyClient.recordListBySubdomain(createTestZone(), "www", RecordType.TXT); - verify(spyClient).getRequest(eq("/zones/" + TEST_ZONE_ID + "/dns_records?type=MX&page=1&per_page=" + verify(spyClient).getRequest(eq("/zones/" + TEST_ZONE_ID + "/dns_records?type=MX&order=name&direction=asc&page=1&per_page=" + CfDnsClient.RECORDS_PER_PAGE), eq(RecordMultipleResponse.class)); verify(spyClient).getRequest(eq("/zones/" + TEST_ZONE_ID - + "/dns_records?name=www.example.com&type=TXT&page=1&per_page=" + + "/dns_records?name=www.example.com&type=TXT&order=name&direction=asc&page=1&per_page=" + CfDnsClient.RECORDS_PER_PAGE), eq(RecordMultipleResponse.class)); } @@ -624,7 +624,7 @@ class CfDnsClientMockTest { List records = spyClient.recordListByType(createTestZone(), RecordType.A, RecordType.AAAA); assertEquals(List.of("A", "AAAA"), records.stream().map(RecordEntity::getType).toList()); - verify(spyClient).getRequest(eq("/zones/" + TEST_ZONE_ID + "/dns_records?page=1&per_page=" + verify(spyClient).getRequest(eq("/zones/" + TEST_ZONE_ID + "/dns_records?order=name&direction=asc&page=1&per_page=" + CfDnsClient.RECORDS_PER_PAGE), eq(RecordMultipleResponse.class)); } @@ -759,10 +759,10 @@ class CfDnsClientMockTest { spyClient.recordListBySubdomain(createTestZone(), "0/26"); verify(spyClient).getRequest(eq("/zones/" + TEST_ZONE_ID - + "/dns_records?name=%24%28reboot%29.example.com&page=1&per_page=" + CfDnsClient.RECORDS_PER_PAGE), + + "/dns_records?name=%24%28reboot%29.example.com&order=name&direction=asc&page=1&per_page=" + CfDnsClient.RECORDS_PER_PAGE), eq(RecordMultipleResponse.class)); verify(spyClient).getRequest(eq("/zones/" + TEST_ZONE_ID - + "/dns_records?name=0%2F26.example.com&page=1&per_page=" + CfDnsClient.RECORDS_PER_PAGE), + + "/dns_records?name=0%2F26.example.com&order=name&direction=asc&page=1&per_page=" + CfDnsClient.RECORDS_PER_PAGE), eq(RecordMultipleResponse.class)); } @@ -779,4 +779,49 @@ class CfDnsClientMockTest { verify(spyClient, never()).patchRequest(anyString(), any(), any()); verify(spyClient, never()).getRequest(anyString(), any()); } + + private static RecordEntity rec(String id, String name) { + RecordEntity rec = RecordEntity.build(name, RecordType.A, 60, "1.1.1.1"); + rec.setId(id); + return rec; + } + + private static String recordsPage(int page) { + return "/zones/" + TEST_ZONE_ID + "/dns_records?order=name&direction=asc&page=" + page + + "&per_page=" + CfDnsClient.RECORDS_PER_PAGE; + } + + @Test + void testRecordList_InconsistentPagesAreFetchedAgain() throws Exception { + CfDnsClient spyClient = spy(client); + // 1st attempt: page 2 repeats a record of page 1 (unstable order), so record c is missing + doReturn(recordResponse(List.of(rec("a", "a.example.com"), rec("b", "b.example.com")), + new ResultInfo(1, 2, 2, 2, 3))) + .when(spyClient).getRequest(eq(recordsPage(1)), eq(RecordMultipleResponse.class)); + doReturn(recordResponse(List.of(rec("b", "b.example.com")), new ResultInfo(2, 2, 2, 1, 3)), + recordResponse(List.of(rec("c", "c.example.com")), new ResultInfo(2, 2, 2, 1, 3))) + .when(spyClient).getRequest(eq(recordsPage(2)), eq(RecordMultipleResponse.class)); + + List records = spyClient.recordList(createTestZone()); + + assertEquals(List.of("a", "b", "c"), records.stream().map(RecordEntity::getId).toList()); + verify(spyClient, times(2)).getRequest(eq(recordsPage(1)), eq(RecordMultipleResponse.class)); + verify(spyClient, times(2)).getRequest(eq(recordsPage(2)), eq(RecordMultipleResponse.class)); + } + + @Test + void testRecordList_PersistentlyInconsistentPagesReturnDeduplicatedResult() throws Exception { + CfDnsClient spyClient = spy(client); + doReturn(recordResponse(List.of(rec("a", "a.example.com"), rec("b", "b.example.com")), + new ResultInfo(1, 2, 2, 2, 3))) + .when(spyClient).getRequest(eq(recordsPage(1)), eq(RecordMultipleResponse.class)); + doReturn(recordResponse(List.of(rec("b", "b.example.com")), new ResultInfo(2, 2, 2, 1, 3))) + .when(spyClient).getRequest(eq(recordsPage(2)), eq(RecordMultipleResponse.class)); + + List records = spyClient.recordList(createTestZone()); + + // no duplicates, after 3 attempts the incomplete result is returned + assertEquals(List.of("a", "b"), records.stream().map(RecordEntity::getId).toList()); + verify(spyClient, times(3)).getRequest(eq(recordsPage(1)), eq(RecordMultipleResponse.class)); + } }