From 2fd63e734b2f545d9b55b8e5d9a90820231826c4 Mon Sep 17 00:00:00 2001 From: James Maes Date: Fri, 6 Nov 2026 08:46:05 -0600 Subject: [PATCH 1/3] fix(sync): resolve country and state foreign keys and fail on rejected rows Port the sync-step half of GG Forge's qualified geo-data-sync.patch (its entity/producer half landed in b5787bb). The 0.1.0 step matched states on countryAlpha2 + code and cities on countryAlpha2 + stateCode + name, columns that do not exist in the schema, so it reported success after persisting 250 countries and zero states or cities. Resolve the bundled source codes to the persisted countryId and stateProvinceId before matching; match cities on stateProvinceId, name, latitude and longitude at the schema's seven-decimal precision; compare numbers by BigDecimal value so a second run is a no-op; reactivate an inactive row whose source entry returns; and check insert/update results so a row the backend rejects fails the sync with a QException. Same change as the Forge pin 0.1.0-forge.6ac6a0366c34-sync1, with the inline java.math qualifications turned into imports and the class Javadoc updated. Co-Authored-By: Claude Fable 5.1 Signed-off-by: James Maes --- .../qbits/geodata/sync/GeoDataSyncStep.java | 73 +++++++++++++++---- 1 file changed, 60 insertions(+), 13 deletions(-) diff --git a/qbit-geo-data-core/src/main/java/com/kingsrook/qbits/geodata/sync/GeoDataSyncStep.java b/qbit-geo-data-core/src/main/java/com/kingsrook/qbits/geodata/sync/GeoDataSyncStep.java index a14eb96..b108a5c 100644 --- a/qbit-geo-data-core/src/main/java/com/kingsrook/qbits/geodata/sync/GeoDataSyncStep.java +++ b/qbit-geo-data-core/src/main/java/com/kingsrook/qbits/geodata/sync/GeoDataSyncStep.java @@ -4,13 +4,15 @@ ** Syncs all three entity types (Country, StateProvince, City) from bundled ** JSON files into database tables. Performs natural key upsert: ** - Country: by alpha2Code - ** - StateProvince: by countryAlpha2 + code - ** - City: by countryAlpha2 + stateCode + name + ** - StateProvince: by countryId + code (countryAlpha2 resolved to the persisted country) + ** - City: by stateProvinceId + name + latitude + longitude (codes resolved to the persisted state) *******************************************************************************/ package com.kingsrook.qbits.geodata.sync; import java.io.InputStream; +import java.math.BigDecimal; +import java.math.RoundingMode; import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.HashMap; @@ -62,9 +64,39 @@ public class GeoDataSyncStep implements BackendStep // Sync in order: countries first (no dependencies), then states, then // // cities (which may reference states) // ////////////////////////////////////////////////////////////////////////// - int countriesInserted = syncTable(prefix + "_country", "/data/countries.json", List.of("alpha2Code")); - int statesInserted = syncTable(prefix + "_stateProvince", "/data/states.json", List.of("countryAlpha2", "code")); - int citiesInserted = syncTable(prefix + "_city", "/data/cities.json", List.of("countryAlpha2", "stateCode", "name")); + int countriesInserted = syncTable(prefix + "_country", loadJsonData("/data/countries.json"), List.of("alpha2Code")); + Map countries = queryExisting(prefix + "_country", List.of("alpha2Code")); + List states = loadJsonData("/data/states.json"); + for(QRecord state : states) + { + QRecord country = countries.get(state.getValueString("countryAlpha2")); + if(country == null) + { + throw new QException("Geo Data state references an unavailable country"); + } + state.setValue("countryId", country.getValue("id")); + state.getValues().remove("countryAlpha2"); + } + int statesInserted = syncTable(prefix + "_stateProvince", states, List.of("countryId", "code")); + Map persistedStates = queryExisting(prefix + "_stateProvince", List.of("countryId", "code")); + List cities = loadJsonData("/data/cities.json"); + for(QRecord city : cities) + { + QRecord country = countries.get(city.getValueString("countryAlpha2")); + QRecord state = country == null ? null : persistedStates.get(country.getValue("id") + "|" + city.getValueString("stateCode")); + if(state == null) + { + throw new QException("Geo Data city references an unavailable state"); + } + city.setValue("stateProvinceId", state.getValue("id")); + for(String coordinate : List.of("latitude", "longitude")) + { + city.setValue(coordinate, new BigDecimal(city.getValue(coordinate).toString()).setScale(7, RoundingMode.HALF_UP)); + } + city.getValues().remove("countryAlpha2"); + city.getValues().remove("stateCode"); + } + int citiesInserted = syncTable(prefix + "_city", cities, List.of("stateProvinceId", "name", "latitude", "longitude")); LOG.info("Geo data sync complete", logPair("prefix", prefix), @@ -83,17 +115,15 @@ public class GeoDataSyncStep implements BackendStep ** Sync a single table from JSON resource. ** Returns count of records processed. *******************************************************************************/ - private int syncTable(String tableName, String resourcePath, List naturalKeyFields) throws QException + private int syncTable(String tableName, List sourceRecords, List naturalKeyFields) throws QException { LOG.info("Syncing table", logPair("table", tableName), - logPair("resource", resourcePath), logPair("naturalKey", naturalKeyFields)); ///////////////////////////////////////////////////////////////////////// // 1. Load source data from classpath JSON // ///////////////////////////////////////////////////////////////////////// - List sourceRecords = loadJsonData(resourcePath); LOG.info("Loaded source data", logPair("count", sourceRecords.size())); ///////////////////////////////////////////////////////////////////////// @@ -119,7 +149,7 @@ public class GeoDataSyncStep implements BackendStep source.setValue("isActive", true); toInsert.add(source); } - else if(hasChanges(source, existing)) + else if(hasChanges(source, existing) || !Boolean.TRUE.equals(existing.getValueBoolean("isActive"))) { source.setValue("id", existing.getValue("id")); source.setValue("isActive", true); @@ -182,7 +212,7 @@ public class GeoDataSyncStep implements BackendStep key.append("|"); } Object value = record.getValue(field); - key.append(value != null ? value.toString() : ""); + key.append(value instanceof Number ? new BigDecimal(value.toString()).stripTrailingZeros().toPlainString() : value != null ? value.toString() : ""); } return key.toString(); } @@ -265,7 +295,14 @@ public class GeoDataSyncStep implements BackendStep } Object sourceValue = source.getValue(fieldName); Object existingValue = existing.getValue(fieldName); - if(!Objects.equals(sourceValue, existingValue)) + if(sourceValue instanceof Number && existingValue instanceof Number) + { + if(new BigDecimal(sourceValue.toString()).compareTo(new BigDecimal(existingValue.toString())) != 0) + { + return true; + } + } + else if(!Objects.equals(sourceValue, existingValue)) { return true; } @@ -283,7 +320,7 @@ public class GeoDataSyncStep implements BackendStep InsertInput insertInput = new InsertInput(); insertInput.setTableName(tableName); insertInput.setRecords(records); - new InsertAction().execute(insertInput); + requireSuccess(tableName, records.size(), new InsertAction().execute(insertInput).getRecords()); } @@ -296,6 +333,16 @@ public class GeoDataSyncStep implements BackendStep UpdateInput updateInput = new UpdateInput(); updateInput.setTableName(tableName); updateInput.setRecords(records); - new UpdateAction().execute(updateInput); + requireSuccess(tableName, records.size(), new UpdateAction().execute(updateInput).getRecords()); + } + /******************************************************************************* + ** Do not report a successful sync when a native action rejected rows. + *******************************************************************************/ + private void requireSuccess(String tableName, int expected, List records) throws QException + { + if(records == null || records.size() != expected || records.stream().anyMatch(record -> !record.getErrors().isEmpty())) + { + throw new QException("Geo Data persistence failed for table " + tableName); + } } } -- 2.54.0 From 401e7ca4712d6cc46a1cba03635aa03a4f94c68c Mon Sep 17 00:00:00 2001 From: James Maes Date: Fri, 6 Nov 2026 08:46:06 -0600 Subject: [PATCH 2/3] chore(release): prepare 0.2.0 Set the revision to 0.2.0: main already started the 0.2.0 line (6337bfe) because the QQQ dependency moved from 0.35.0 to 4.1.0-RC.21, a minor change, not a patch. Align GeoDataQBitProducer.VERSION and its test with the pom. Add the 0.2.0 changelog entry, move the already-shipped 0.1.0 bullets out of Unreleased, and correct the README coordinates (artifactId qbit-geo-data, version 0.2.0, Gitea registry, Java 21 / QQQ 4.1). No tag, release or publish: a push to main only verifies; the v0.2.0 tag publishes through the shared ci-workflows Maven workflow. Co-Authored-By: Claude Fable 5.1 Signed-off-by: James Maes --- CHANGELOG.md | 33 +++++++++++++++++++ README.md | 21 ++++++++++-- pom.xml | 2 +- .../qbits/geodata/GeoDataQBitProducer.java | 2 +- .../geodata/GeoDataQBitProducerTest.java | 2 +- 5 files changed, 55 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fea915e..d9471ad 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,39 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.2.0] - 2026-11-06 + +Published as `com.kingsrook.qbits:qbit-geo-data:0.2.0` on the QRun-IO Gitea Maven +registry. Requires Java 21 and QQQ 4.1 (built against `qqq-bom-pom` 4.1.0-RC.21). +This is the first release that works on QQQ 4.1; 0.1.0 was built against QQQ 0.35.0. + +### Changed +- Build against the published QQQ 4.1 core (`qqq-bom-pom` 4.1.0-RC.21) instead of 0.35.0. +- `GeoDataQBitProducer.VERSION` is `0.2.0`. +- Import the project's `junit-bom` 5.10.0 ahead of the QQQ BOM so every JUnit artifact + converges on 5.10.0 and Surefire discovers the tests. +- CI runs on Gitea through the shared `KofTwentyTwo/ci-workflows` Maven workflow + (`main` verifies; a `vX.Y.Z` tag publishes to the Gitea registry). + +### Fixed +- `Country`, `StateProvince` and `City` ask for their tables (`produceTableMetaData`) + and expose the setters `QRecordEntity` needs to discover fields, so the producer + creates `_country`, `_stateProvince` and `_city` on core 4.1 + instead of zero tables. The producer tolerates a table without fields. +- `GeoDataSyncStep` resolves the bundled `countryAlpha2` / `stateCode` source codes to + the persisted `countryId` / `stateProvinceId` before matching, so states and cities + are persisted (0.1.0 reported success after writing 250 countries and zero states or + cities because those source columns do not exist in the schema). +- Cities match on `stateProvinceId`, `name`, `latitude` and `longitude` at the schema's + seven-decimal precision; city names are not unique within a subdivision. +- Numeric comparisons use `BigDecimal` value equality, so a second run reports zero + inserts, updates or deactivations. +- An inactive existing row whose source entry returns is reactivated. +- Insert and update results are checked; a row rejected by the backend fails the sync + with a `QException` instead of reporting success. + +## [0.1.0] - 2025-12-28 + ### Added - Multi-module structure (qbit-geo-data-core + tools) - Country, StateProvince, City entities with sync process diff --git a/README.md b/README.md index e6bd4d0..ab31bc9 100644 --- a/README.md +++ b/README.md @@ -7,16 +7,33 @@ Geographic reference data QBit providing countries, states/provinces, and cities ## Quick Start +The current release is **0.2.0**. It requires Java 21 and QQQ 4.1 (built against +`qqq-bom-pom` 4.1.0-RC.21). See [CHANGELOG.md](CHANGELOG.md). + Add the dependency to your `pom.xml`: ```xml com.kingsrook.qbits - qbit-geo-data-core - ${qbit-geo-data.version} + qbit-geo-data + 0.2.0 ``` +Releases are published to the QRun-IO Gitea Maven registry, which requires +authentication for reads as well as writes. Add the repository and a matching +`` entry (with your own Gitea username and token) in your private Maven +`settings.xml`: + +```xml + + qrun-io + https://git.kof22.com/api/packages/QRun-IO/maven + true + false + +``` + ## What You Get This QBit provides three entities with pre-populated reference data: diff --git a/pom.xml b/pom.xml index 4906a7e..fa48752 100644 --- a/pom.xml +++ b/pom.xml @@ -50,7 +50,7 @@ - 0.2.0-SNAPSHOT + 0.2.0 4.1.0-RC.21 21 21 diff --git a/qbit-geo-data-core/src/main/java/com/kingsrook/qbits/geodata/GeoDataQBitProducer.java b/qbit-geo-data-core/src/main/java/com/kingsrook/qbits/geodata/GeoDataQBitProducer.java index 80b2a0e..bda7273 100644 --- a/qbit-geo-data-core/src/main/java/com/kingsrook/qbits/geodata/GeoDataQBitProducer.java +++ b/qbit-geo-data-core/src/main/java/com/kingsrook/qbits/geodata/GeoDataQBitProducer.java @@ -28,7 +28,7 @@ public class GeoDataQBitProducer implements QBitProducer { public static final String GROUP_ID = "com.kingsrook.qbits"; public static final String ARTIFACT_ID = "qbit-geo-data"; - public static final String VERSION = "0.1.0"; + public static final String VERSION = "0.2.0"; private GeoDataQBitConfig config; diff --git a/qbit-geo-data-core/src/test/java/com/kingsrook/qbits/geodata/GeoDataQBitProducerTest.java b/qbit-geo-data-core/src/test/java/com/kingsrook/qbits/geodata/GeoDataQBitProducerTest.java index 136347b..a31b88d 100644 --- a/qbit-geo-data-core/src/test/java/com/kingsrook/qbits/geodata/GeoDataQBitProducerTest.java +++ b/qbit-geo-data-core/src/test/java/com/kingsrook/qbits/geodata/GeoDataQBitProducerTest.java @@ -165,6 +165,6 @@ class GeoDataQBitProducerTest { assertThat(GeoDataQBitProducer.GROUP_ID).isEqualTo("com.kingsrook.qbits"); assertThat(GeoDataQBitProducer.ARTIFACT_ID).isEqualTo("qbit-geo-data"); - assertThat(GeoDataQBitProducer.VERSION).isEqualTo("0.1.0"); + assertThat(GeoDataQBitProducer.VERSION).isEqualTo("0.2.0"); } } -- 2.54.0 From b50515a8e82f8852d3b655ff405e516c9a9e407c Mon Sep 17 00:00:00 2001 From: James Maes Date: Fri, 6 Nov 2026 08:51:21 -0600 Subject: [PATCH 3/3] build(pom): flatten the CI-friendly revision in published POMs The root pom keeps its version in the CI-friendly ${revision} property and the shared ci-workflows Maven workflow deploys with a plain mvn deploy, so the installed and deployed module POMs carried an unresolved ${revision} that a consuming build cannot resolve (the retired CircleCI orb path published resolved POMs, 68e85e6). Add flatten-maven-plugin 1.7.3 in resolveCiFriendliesOnly mode so the published POMs carry the literal version; ignore the generated .flattened-pom.xml. Verified with Maven 3.9.11: clean verify passes (29 tests), the installed qbit-geo-data-0.2.0.pom declares parent version 0.2.0, and a throwaway consumer resolves com.kingsrook.qbits:qbit-geo-data:0.2.0 through the full parent chain. Co-Authored-By: Claude Fable 5.1 Signed-off-by: James Maes --- .gitignore | 1 + pom.xml | 29 +++++++++++++++++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/.gitignore b/.gitignore index 7c97728..e7883b3 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ # Maven target/ +.flattened-pom.xml *.jar *.war *.ear diff --git a/pom.xml b/pom.xml index fa48752..9e557d2 100644 --- a/pom.xml +++ b/pom.xml @@ -92,6 +92,35 @@ + + + + + org.codehaus.mojo + flatten-maven-plugin + 1.7.3 + + true + resolveCiFriendliesOnly + + + + flatten + process-resources + + flatten + + + + flatten-clean + clean + + clean + + + + + -- 2.54.0