From 8608ee3ce131f976fe0c2d044eb2350a821dbab0 Mon Sep 17 00:00:00 2001 From: Thilo Schwarz Date: Mon, 6 Jul 2026 19:40:42 +0200 Subject: [PATCH] Migrate Git-based repository management to Maven-based artifact handling and refactor services accordingly --- .claude/settings.local.json | 4 +- pom.xml | 2 + .../thischwa/jdvs/config/JdvsConfig.java | 7 +- .../thischwa/jdvs/model/GitRepository.java | 3 - .../thischwa/jdvs/service/GitService.java | 63 ---- .../thischwa/jdvs/service/JavadocService.java | 115 ------- .../jdvs/service/MavenSourceService.java | 290 ++++++++++++++++++ .../jdvs/service/RepoConfigLoader.java | 1 - .../jdvs/service/UpdateScheduler.java | 20 +- src/main/resources/application.yml | 1 + .../db/changelog/001-init-schema.yaml | 5 - src/main/resources/templates/index.html | 6 +- 12 files changed, 313 insertions(+), 204 deletions(-) delete mode 100644 src/main/java/codes/thischwa/jdvs/service/GitService.java delete mode 100644 src/main/java/codes/thischwa/jdvs/service/JavadocService.java create mode 100644 src/main/java/codes/thischwa/jdvs/service/MavenSourceService.java diff --git a/.claude/settings.local.json b/.claude/settings.local.json index cac083c..5c4628f 100644 --- a/.claude/settings.local.json +++ b/.claude/settings.local.json @@ -13,7 +13,9 @@ "Bash(sdk use:*)", "Bash(mvn clean:*)", "Bash(source \"$HOME/.sdkman/bin/sdkman-init.sh\")", - "Bash(sed -i '' 's/21172117 21 + ${java.version} + ${java.version} UTF-8 diff --git a/src/main/java/codes/thischwa/jdvs/config/JdvsConfig.java b/src/main/java/codes/thischwa/jdvs/config/JdvsConfig.java index 1e545cb..295ce1e 100644 --- a/src/main/java/codes/thischwa/jdvs/config/JdvsConfig.java +++ b/src/main/java/codes/thischwa/jdvs/config/JdvsConfig.java @@ -1,7 +1,9 @@ package codes.thischwa.jdvs.config; import java.util.List; +import java.util.Objects; import lombok.Data; +import org.jspecify.annotations.Nullable; import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.context.annotation.Configuration; @@ -12,11 +14,14 @@ public class JdvsConfig { private String baseDir; private boolean cleanOnStart; private boolean runOnStart; + private String mavenCentralUrl; private List repositories; @Data public static class RepoConfig { private String name; - private String url; + private String groupId; + private String artifactId; + private String mavenRepoUrl; } } diff --git a/src/main/java/codes/thischwa/jdvs/model/GitRepository.java b/src/main/java/codes/thischwa/jdvs/model/GitRepository.java index 1dc0fb4..9af0c26 100644 --- a/src/main/java/codes/thischwa/jdvs/model/GitRepository.java +++ b/src/main/java/codes/thischwa/jdvs/model/GitRepository.java @@ -18,9 +18,6 @@ public class GitRepository { @Column(nullable = false, unique = true) private String name; - @Column(nullable = false) - private String url; - private String lastTag; private LocalDateTime updated; diff --git a/src/main/java/codes/thischwa/jdvs/service/GitService.java b/src/main/java/codes/thischwa/jdvs/service/GitService.java deleted file mode 100644 index 5fe606b..0000000 --- a/src/main/java/codes/thischwa/jdvs/service/GitService.java +++ /dev/null @@ -1,63 +0,0 @@ -package codes.thischwa.jdvs.service; - -import codes.thischwa.jdvs.config.JdvsConfig; -import codes.thischwa.jdvs.model.GitRepository; -import java.io.File; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.Comparator; -import java.util.List; -import java.util.Optional; -import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; -import org.eclipse.jgit.api.Git; -import org.eclipse.jgit.api.errors.GitAPIException; -import org.eclipse.jgit.lib.Ref; -import org.springframework.stereotype.Service; - -@Service -@RequiredArgsConstructor -@Slf4j -public class GitService { - private final JdvsConfig jdvsConfig; - - public Optional updateAndCheckoutLatestTag(GitRepository repoEntity) { - Path repoPath = Path.of(jdvsConfig.getBaseDir(), "repos", repoEntity.getName()); - try { - Git git; - if (Files.exists(repoPath)) { - git = Git.open(repoPath.toFile()); - git.fetch().setCheckFetchedObjects(true).call(); - } else { - git = Git.cloneRepository() - .setURI(repoEntity.getUrl()) - .setDirectory(repoPath.toFile()) - .setCloneAllBranches(true) - .setNoCheckout(false) - .call(); - } - - List tags = git.tagList().call(); - Optional latestVTag = tags.stream() - .filter(ref -> ref.getName().startsWith("refs/tags/v")) - .max(Comparator.comparing(Ref::getName)); - - if (latestVTag.isPresent()) { - String tagName = latestVTag.get().getName().substring("refs/tags/".length()); - git.checkout().setName(tagName).call(); - log.info("Repository {} checked out at tag {}.", repoEntity.getName(), tagName); - return Optional.of(tagName); - } else { - log.warn("No v* tag found for repository {}.", repoEntity.getName()); - } - } catch (IOException | GitAPIException e) { - log.error("Error processing repository " + repoEntity.getName(), e); - } - return Optional.empty(); - } - - public File getRepoDirectory(String name) { - return Path.of(jdvsConfig.getBaseDir(), "repos", name).toFile(); - } -} diff --git a/src/main/java/codes/thischwa/jdvs/service/JavadocService.java b/src/main/java/codes/thischwa/jdvs/service/JavadocService.java deleted file mode 100644 index e73a760..0000000 --- a/src/main/java/codes/thischwa/jdvs/service/JavadocService.java +++ /dev/null @@ -1,115 +0,0 @@ -package codes.thischwa.jdvs.service; - -import codes.thischwa.jdvs.config.JdvsConfig; -import java.io.BufferedReader; -import java.io.File; -import java.io.IOException; -import java.io.InputStreamReader; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.StandardCopyOption; -import java.util.stream.Stream; -import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; -import org.springframework.stereotype.Service; - -@Service -@RequiredArgsConstructor -@Slf4j -public class JavadocService { - private final JdvsConfig jdvsConfig; - // Possible standard paths for Javadoc - private final String[] possiblePaths = { - "target/reports/apidocs", - "target/site/apidocs", - "target/apidocs" - }; - - public boolean generateJavadoc(String projectName, File repoDir) { - File outputDir = Path.of(jdvsConfig.getBaseDir(), "javadoc", projectName).toFile(); - if (!outputDir.exists()) { - outputDir.mkdirs(); - } - - // Try Maven first if a pom.xml is present - if (new File(repoDir, "pom.xml").exists()) { - boolean success = runMavenJavadoc(repoDir, outputDir); - if (success) { - copyGeneratedJavadoc(repoDir, outputDir); - } - return success; - } else { - log.warn("No pom.xml found in {}. Manual javadoc generation is not implemented.", projectName); - return false; - } - } - - private void copyGeneratedJavadoc(File repoDir, File targetDir) { - for (String relPath : possiblePaths) { - File sourceDir = new File(repoDir, relPath); - if (sourceDir.exists() && sourceDir.isDirectory()) { - log.info("Found Javadoc in {}, copying to {}", sourceDir.getAbsolutePath(), targetDir.getAbsolutePath()); - try { - copyDirectory(sourceDir.toPath(), targetDir.toPath()); - return; - } catch (IOException e) { - log.error("Error copying Javadoc from {} to {}", sourceDir.getAbsolutePath(), targetDir.getAbsolutePath(), e); - } - } - } - log.warn("No generated Javadoc files found in the standard directories of {}.", repoDir.getName()); - } - - private void copyDirectory(Path source, Path target) throws IOException { - try (Stream stream = Files.walk(source)) { - stream.forEach(path -> { - try { - Path dest = target.resolve(source.relativize(path)); - if (Files.isDirectory(path)) { - if (!Files.exists(dest)) { - Files.createDirectories(dest); - } - } else { - Files.copy(path, dest, StandardCopyOption.REPLACE_EXISTING); - } - } catch (IOException e) { - throw new RuntimeException(e); - } - }); - } - } - - private boolean runMavenJavadoc(File repoDir, File outputDir) { - ProcessBuilder pb = new ProcessBuilder( - "mvn", "javadoc:javadoc", - "-Dmaven.javadoc.failOnError=false", - "--no-transfer-progress", - "-Dlombok.delombok.skip=true", - "-Dcheckstyle.skip=true", - "-Djacoco.skip=true", - "-DskipTests" - ); - pb.directory(repoDir); - pb.redirectErrorStream(true); - - try { - Process process = pb.start(); - try (BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream()))) { - String line; - while ((line = reader.readLine()) != null) { - log.info("[MAVEN] {}", line); - } - } - int exitCode = process.waitFor(); - if (exitCode == 0) { - log.info("Javadoc successfully generated in {}", outputDir.getAbsolutePath()); - return true; - } else { - log.error("Maven javadoc failed with exit code {}", exitCode); - } - } catch (IOException | InterruptedException e) { - log.error("Error executing Maven javadoc", e); - } - return false; - } -} diff --git a/src/main/java/codes/thischwa/jdvs/service/MavenSourceService.java b/src/main/java/codes/thischwa/jdvs/service/MavenSourceService.java new file mode 100644 index 0000000..0b74ec0 --- /dev/null +++ b/src/main/java/codes/thischwa/jdvs/service/MavenSourceService.java @@ -0,0 +1,290 @@ +package codes.thischwa.jdvs.service; + +import codes.thischwa.jdvs.config.JdvsConfig; +import codes.thischwa.jdvs.config.JdvsConfig.RepoConfig; +import java.io.BufferedReader; +import java.io.File; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.StringReader; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.zip.ZipEntry; +import java.util.zip.ZipInputStream; +import javax.xml.parsers.DocumentBuilderFactory; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; +import org.springframework.web.client.RestClient; +import org.w3c.dom.Document; +import org.w3c.dom.NodeList; +import org.xml.sax.InputSource; + +@Service +@RequiredArgsConstructor +@Slf4j +public class MavenSourceService { + + private final JdvsConfig jdvsConfig; + private final RestClient restClient = RestClient.create(); + + public Optional fetchLatestVersion(String repoName) { + RepoConfig cfg = findConfig(repoName); + String metadataUrl = baseUrl(cfg) + "/maven-metadata.xml"; + log.info("Fetching Maven metadata from {}", metadataUrl); + try { + String xml = restClient.get().uri(metadataUrl).retrieve().body(String.class); + Document doc = DocumentBuilderFactory.newInstance() + .newDocumentBuilder() + .parse(new InputSource(new StringReader(xml))); + String version = firstText(doc, "release"); + if (version == null) { + version = firstText(doc, "latest"); + } + if (version != null) { + log.info("Latest version of {} is {}", repoName, version); + return Optional.of(version); + } + log.warn("Could not determine latest version for {}", repoName); + } catch (Exception e) { + log.error("Failed to fetch Maven metadata for {}", repoName, e); + } + return Optional.empty(); + } + + public boolean generateJavadoc(String repoName, String version) { + RepoConfig cfg = findConfig(repoName); + String resolvedVersion = version.endsWith("-SNAPSHOT") + ? resolveSnapshotVersion(cfg, version) + : version; + String jarUrl = baseUrl(cfg) + "/" + version + "/" + + cfg.getArtifactId() + "-" + resolvedVersion + "-sources.jar"; + log.info("Downloading sources JAR from {}", jarUrl); + + Path workDir = null; + try { + workDir = Files.createTempDirectory("jdvs-maven-" + repoName + "-"); + Path srcMainJava = workDir.resolve("src/main/java"); + Files.createDirectories(srcMainJava); + downloadAndExtract(jarUrl, workDir, srcMainJava); + copyPomFromMetaInf(workDir, cfg); + + File outputDir = Path.of(jdvsConfig.getBaseDir(), "javadoc", repoName).toFile(); + if (!outputDir.exists()) { + outputDir.mkdirs(); + } + return runMavenJavadoc(workDir, cfg, repoName, outputDir.toPath()); + } catch (Exception e) { + log.error("Failed to generate Javadoc for {} version {}", repoName, version, e); + return false; + } finally { + if (workDir != null) { + deleteDirectory(workDir); + } + } + } + + private void downloadAndExtract(String jarUrl, Path metaDir, Path srcDir) throws IOException { + byte[] jarBytes = restClient.get().uri(jarUrl).retrieve().body(byte[].class); + if (jarBytes == null) { + throw new IOException("Empty response for " + jarUrl); + } + try (ZipInputStream zip = new ZipInputStream(new java.io.ByteArrayInputStream(jarBytes))) { + ZipEntry entry; + while ((entry = zip.getNextEntry()) != null) { + if (!entry.isDirectory()) { + String name = entry.getName(); + // META-INF goes to metaDir, Java sources go to srcDir + Path base = name.startsWith("META-INF/") ? metaDir : srcDir; + Path dest = base.resolve(name).normalize(); + if (!dest.startsWith(metaDir) && !dest.startsWith(srcDir)) { + throw new IOException("Zip slip detected: " + name); + } + Files.createDirectories(dest.getParent()); + Files.copy(zip, dest, StandardCopyOption.REPLACE_EXISTING); + } + zip.closeEntry(); + } + } + } + + private void copyPomFromMetaInf(Path workDir, RepoConfig cfg) throws IOException { + Path pomSource = workDir.resolve("META-INF/maven") + .resolve(cfg.getGroupId()) + .resolve(cfg.getArtifactId()) + .resolve("pom.xml"); + if (!Files.exists(pomSource)) { + throw new IOException("pom.xml not found in META-INF at " + pomSource); + } + Files.copy(pomSource, workDir.resolve("pom.xml"), StandardCopyOption.REPLACE_EXISTING); + log.info("Copied POM from META-INF to working directory"); + } + + private boolean runMavenJavadoc(Path workDir, RepoConfig cfg, String repoName, Path outputDir) + throws IOException, InterruptedException { + Path mavenLocalRepo = Path.of(jdvsConfig.getBaseDir(), "maven-repo").toAbsolutePath(); + Files.createDirectories(mavenLocalRepo); + + List cmd = new ArrayList<>(List.of( + "mvn", "-U", "--no-transfer-progress", + "clean", "javadoc:javadoc", + "-Djacoco.skip=true", + "-DskipTests", + "-Dcheckstyle.skip=true", + "-Dlombok.delombok.skip=true", + "-Dmaven.repo.local=" + mavenLocalRepo + )); + if (cfg.getMavenRepoUrl() != null) { + cmd.add("-s"); + cmd.add(writeSettingsXml(workDir, cfg.getMavenRepoUrl()).toAbsolutePath().toString()); + } + + ProcessBuilder pb = new ProcessBuilder(cmd); + pb.directory(workDir.toFile()); + pb.redirectErrorStream(true); + Process process = pb.start(); + try (BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream()))) { + String line; + while ((line = reader.readLine()) != null) { + log.debug("[MVN] {}", line); + } + } + int exitCode = process.waitFor(); + if (exitCode != 0) { + log.error("mvn javadoc:javadoc failed with exit code {} for {}", exitCode, repoName); + return false; + } + + Path apidocsDir = workDir.resolve("target/reports/apidocs"); + if (!Files.exists(apidocsDir)) { + // Log actual target content to find where Maven put the output + Path targetDir = workDir.resolve("target"); + if (Files.exists(targetDir)) { + try (var s = Files.walk(targetDir, 4)) { + s.filter(p -> p.toString().endsWith(".html")) + .findFirst() + .ifPresentOrElse( + p -> log.error("Javadoc not at expected path. Found HTML at: {}", p), + () -> log.error("No HTML files found under {}", targetDir) + ); + } + } else { + log.error("target/ directory does not exist in {}", workDir); + } + return false; + } + copyDirectory(apidocsDir, outputDir); + log.info("Javadoc successfully generated for {} in {}", repoName, outputDir); + return true; + } + + private Path writeSettingsXml(Path workDir, String repoUrl) throws IOException { + Path settingsFile = workDir.resolve("settings.xml"); + String xml = """ + + + + + jdvs-repos + + + custom + """ + repoUrl + """ + + true + + + + + + jdvs-repos + + + """; + Files.writeString(settingsFile, xml); + return settingsFile; + } + + private void copyDirectory(Path source, Path target) throws IOException { + try (var stream = Files.walk(source)) { + stream.forEach(path -> { + try { + Path dest = target.resolve(source.relativize(path)); + if (Files.isDirectory(path)) { + Files.createDirectories(dest); + } else { + Files.createDirectories(dest.getParent()); + Files.copy(path, dest, StandardCopyOption.REPLACE_EXISTING); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + }); + } + } + + private String resolveSnapshotVersion(RepoConfig cfg, String version) { + String metadataUrl = baseUrl(cfg) + "/" + version + "/maven-metadata.xml"; + log.info("Resolving SNAPSHOT version from {}", metadataUrl); + try { + String xml = restClient.get().uri(metadataUrl).retrieve().body(String.class); + Document doc = DocumentBuilderFactory.newInstance() + .newDocumentBuilder() + .parse(new InputSource(new StringReader(xml))); + NodeList nodes = doc.getElementsByTagName("snapshotVersion"); + for (int i = 0; i < nodes.getLength(); i++) { + org.w3c.dom.Element el = (org.w3c.dom.Element) nodes.item(i); + String classifier = firstText(el, "classifier"); + String ext = firstText(el, "extension"); + String value = firstText(el, "value"); + if ("sources".equals(classifier) && "jar".equals(ext) && value != null) { + log.info("Resolved SNAPSHOT version to {}", value); + return value; + } + } + } catch (Exception e) { + log.warn("Could not resolve SNAPSHOT version for {}, falling back to {}", + cfg.getArtifactId(), version, e); + } + return version; + } + + private String baseUrl(RepoConfig cfg) { + String repoUrl = cfg.getMavenRepoUrl() != null + ? cfg.getMavenRepoUrl() + : jdvsConfig.getMavenCentralUrl(); + String groupPath = cfg.getGroupId().replace('.', '/'); + return repoUrl + "/" + groupPath + "/" + cfg.getArtifactId(); + } + + private RepoConfig findConfig(String repoName) { + return jdvsConfig.getRepositories().stream() + .filter(r -> repoName.equals(r.getName())) + .findFirst() + .orElseThrow(() -> new IllegalArgumentException("No config found for: " + repoName)); + } + + private String firstText(Document doc, String tagName) { + NodeList nodes = doc.getElementsByTagName(tagName); + return nodes.getLength() > 0 ? nodes.item(0).getTextContent() : null; + } + + private String firstText(org.w3c.dom.Element parent, String tagName) { + NodeList nodes = parent.getElementsByTagName(tagName); + return nodes.getLength() > 0 ? nodes.item(0).getTextContent() : null; + } + + private void deleteDirectory(Path dir) { + try (var stream = Files.walk(dir)) { + stream.sorted(java.util.Comparator.reverseOrder()) + .map(Path::toFile) + .forEach(File::delete); + } catch (IOException e) { + log.warn("Failed to clean temp directory {}", dir, e); + } + } +} diff --git a/src/main/java/codes/thischwa/jdvs/service/RepoConfigLoader.java b/src/main/java/codes/thischwa/jdvs/service/RepoConfigLoader.java index b78acc9..190578c 100644 --- a/src/main/java/codes/thischwa/jdvs/service/RepoConfigLoader.java +++ b/src/main/java/codes/thischwa/jdvs/service/RepoConfigLoader.java @@ -20,7 +20,6 @@ public class RepoConfigLoader { if (repository.findByName(repoCfg.getName()).isEmpty()) { GitRepository repo = new GitRepository(); repo.setName(repoCfg.getName()); - repo.setUrl(repoCfg.getUrl()); repository.save(repo); log.info("Repository {} added from configuration.", repoCfg.getName()); } diff --git a/src/main/java/codes/thischwa/jdvs/service/UpdateScheduler.java b/src/main/java/codes/thischwa/jdvs/service/UpdateScheduler.java index aefc183..0478279 100644 --- a/src/main/java/codes/thischwa/jdvs/service/UpdateScheduler.java +++ b/src/main/java/codes/thischwa/jdvs/service/UpdateScheduler.java @@ -23,8 +23,7 @@ import org.springframework.stereotype.Service; @Slf4j public class UpdateScheduler { private final GitRepositoryRepository repository; - private final GitService gitService; - private final JavadocService javadocService; + private final MavenSourceService mavenSourceService; private final JdvsConfig jdvsConfig; private final RepoConfigLoader repoConfigLoader; @@ -66,19 +65,18 @@ public class UpdateScheduler { private void updateRepo(GitRepository repo) { log.info("Checking repository: {}", repo.getName()); - Optional latestTag = gitService.updateAndCheckoutLatestTag(repo); - if (latestTag.isPresent()) { - String tag = latestTag.get(); - if (!tag.equals(repo.getLastTag())) { - log.info("New tag {} found for {}. Generating Javadoc...", tag, repo.getName()); - File repoDir = gitService.getRepoDirectory(repo.getName()); - if (javadocService.generateJavadoc(repo.getName(), repoDir)) { - repo.setLastTag(tag); + Optional latestVersion = mavenSourceService.fetchLatestVersion(repo.getName()); + if (latestVersion.isPresent()) { + String version = latestVersion.get(); + if (!version.equals(repo.getLastTag())) { + log.info("New version {} found for {}. Generating Javadoc...", version, repo.getName()); + if (mavenSourceService.generateJavadoc(repo.getName(), version)) { + repo.setLastTag(version); repo.setUpdated(LocalDateTime.now()); repository.save(repo); } } else { - log.info("Repository {} is already up to date ({}).", repo.getName(), tag); + log.info("Repository {} is already up to date ({}).", repo.getName(), version); } } } diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index db4abbe..45a82f3 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -3,6 +3,7 @@ jdvs: cron: 0 0/15 * * * ? clean-on-start: false run-on-start: false + maven-central-url: https://repo1.maven.org/maven2 ## import of the individual configuration settings spring: diff --git a/src/main/resources/db/changelog/001-init-schema.yaml b/src/main/resources/db/changelog/001-init-schema.yaml index 694230a..e77c4c9 100644 --- a/src/main/resources/db/changelog/001-init-schema.yaml +++ b/src/main/resources/db/changelog/001-init-schema.yaml @@ -19,11 +19,6 @@ databaseChangeLog: constraints: nullable: false unique: true - - column: - name: url - type: VARCHAR(512) - constraints: - nullable: false - column: name: last_tag type: VARCHAR(255) diff --git a/src/main/resources/templates/index.html b/src/main/resources/templates/index.html index 58d809c..2a5a1f4 100644 --- a/src/main/resources/templates/index.html +++ b/src/main/resources/templates/index.html @@ -11,8 +11,7 @@ Name - Git URL - Latest Tag + Latest Version Last Update Action @@ -20,8 +19,7 @@ Project - URL - v1.0 + - - Open Javadoc