diff --git a/.circleci/config.yml b/.circleci/config.yml index 8828026..010cd62 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -3,7 +3,19 @@ version: 2.1 orbs: qqq-orb: kingsrook/qqq-orb@0.5.0 +commands: + configure-package-install: + steps: + - run: + name: Keep package installation unattended + command: | + # These build machines are discarded after the job. + sudo install -d /etc/needrestart/conf.d + printf '%s\n' '$nrconf{restart} = "l";' | sudo tee /etc/needrestart/conf.d/99-qqq-ci.conf >/dev/null + workflows: build: jobs: - - qqq-orb/mvn_test + - qqq-orb/mvn_test_only: + pre-steps: + - configure-package-install diff --git a/CLAUDE.md b/CLAUDE.md index 4c3e624..cf97527 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -4,6 +4,10 @@ GitHub template repo for scaffolding QQQ "Data QBits" (reference data + table prefixing + sync process + Liquibase generation). Placeholder package `com.kingsrook.qbits.example`; meant to be renamed by template consumers. +## QQQ 4.0 baseline — 2026-09-24 + +The historical review below predates the 4.0 migration. This template targets Java 21 and the public QQQ 4.0.0 release from Maven Central. The corrected APIs, license references and actual template commands are in README.md and docs/00-getting-started.md. Preserve the documented multi-instance and scaffold limitations. + ## Knowledge base - Domain hub: `R:/Git.Local/KofTwentyTwo/second-brain/knowledge/qqq/qqq-hub.md` diff --git a/README.md b/README.md index f46dde7..ee95bda 100644 --- a/README.md +++ b/README.md @@ -12,24 +12,20 @@ Data QBits provide reusable reference data sets that applications consume. They ## Quick Start -1. Click "Use this template" to create your QBit -2. Run `scripts/customize_template.py` to rename the example -3. Define your entity classes -4. Add reference data JSON files -5. Implement sync process -6. Generate Liquibase changelog +Requires **Java 21**, **Maven 3.8+**, and QQQ **4.0.0**. Create your repository, rename its Maven coordinates and Java package/classes, then run `mvn clean verify`. See [Getting Started](docs/00-getting-started.md). ## Structure ``` src/main/java/com/kingsrook/qbits/example/ -├── ExampleDataQBitConfig.java # Extends AbstractDataQBitConfig -├── ExampleDataQBitProducer.java # Extends AbstractDataQBitProducer +├── ExampleDataQBitConfig.java # Configuration +├── ExampleDataQBitProducer.java # QBit registration ├── model/ │ ├── ExampleEntity.java │ └── ExampleChildEntity.java ├── sync/ -│ └── ExampleDataSyncProcess.java +│ ├── ExampleDataSyncStep.java +│ └── ExampleDataSyncProcessMetaDataProducer.java └── liquibase/ └── ExampleLiquibaseGenerator.java ``` @@ -37,28 +33,20 @@ src/main/java/com/kingsrook/qbits/example/ ## Key Characteristics - **Table prefixing** - `shipping_country`, `billing_country` -- **Multi-instance** - Same QBit deployed multiple times +- **Host configuration** - Backend and optional table prefix - **Sync process** - Upsert by natural key - **Liquibase generator** - Template-based, prefix-aware -## Multi-Instance Usage +## Host integration ```java -// Shipping addresses -new GeoDataQBitProducer() - .withConfig(new GeoDataQBitConfig() - .withBackendName("rdbms") - .withTableNamePrefix("shipping")) - .produce(qInstance, "shipping-geo"); - -// Billing addresses -new GeoDataQBitProducer() - .withConfig(new GeoDataQBitConfig() - .withBackendName("rdbms") - .withTableNamePrefix("billing")) - .produce(qInstance, "billing-geo"); +new ExampleDataQBitProducer() + .withConfig(new ExampleDataQBitConfig().withBackendName("rdbms")) + .produce(qInstance, "example-data"); ``` +The host must already define the backend. The example prefixes table names; adapting it for multiple instances also requires unique process and possible-value-source names and consistent references. See [Getting Started](docs/00-getting-started.md). + ## Documentation - [Getting Started](docs/00-getting-started.md) @@ -68,4 +56,4 @@ new GeoDataQBitProducer() ## License -AGPL-3.0 - See [LICENSE](LICENSE) +See [LICENSE](LICENSE) and [NOTICE](NOTICE). diff --git a/docs/00-getting-started.md b/docs/00-getting-started.md index 047d995..e281ee8 100644 --- a/docs/00-getting-started.md +++ b/docs/00-getting-started.md @@ -1,47 +1,30 @@ # Getting Started -## Prerequisites +Targets Java 21, Maven 3.8 or later, and QQQ 4.0.0. Maven resolves the framework dependencies from Maven Central. -- Java 17+ -- Maven 3.8+ -- QQQ 0.23.0+ +1. Choose **Use this template** on GitHub, then clone the repository you created. +2. In `pom.xml`, set your own `groupId`, `artifactId`, name, description, and version. +3. Use your IDE's package refactoring to rename `com.kingsrook.qbits.example` throughout `src/`. Rename the `Example*` classes and update their imports and references with the IDE's rename refactoring. +4. Update the QBit producer's `GROUP_ID`, `ARTIFACT_ID`, and `VERSION` constants to match your project. Give the metadata names their own stable names before combining the QBit with other examples. +5. Build the generated project: -## Creating Your Data QBit - -1. Click "Use this template" on GitHub -2. Clone your new repository -3. Run the customization script: - ```bash - python scripts/customize_template.py - ``` -4. Enter your QBit name when prompted - -## Project Structure - -``` -src/main/java/com/kingsrook/qbits/yourdata/ -├── YourDataQBitConfig.java -├── YourDataQBitProducer.java -├── model/ -│ └── YourEntity.java -├── sync/ -│ └── YourDataSyncProcess.java -└── liquibase/ - └── YourLiquibaseGenerator.java +```bash +mvn clean verify ``` -## Using Your QBit +The repository provides example Java sources; customization uses normal package/class refactoring. Keep example coordinates unpublished. After customizing, add tests for your QBit's behavior before setting up its publishing workflow. + +## Register with a host application + +The following uses the original example names; substitute the names chosen above: ```java -new YourDataQBitProducer() - .withConfig(new YourDataQBitConfig() - .withBackendName("rdbms") - .withTableNamePrefix("myprefix")) +new ExampleDataQBitProducer() + .withConfig(new ExampleDataQBitConfig() + .withBackendName("rdbms")) .produce(qInstance, "my-data"); ``` -## Next Steps +Register the configured backend in the host first. If you add table prefixes or multiple QBit instances, update related table references, possible-value-source names, and process/widget names consistently. A table prefix alone does not namespace all metadata. -- Read [Data QBit Architecture](01-data-qbit-architecture.md) -- Implement your sync process -- Generate Liquibase changelog +[Data QBit Architecture](01-data-qbit-architecture.md) explains the example data model. diff --git a/docs/01-data-qbit-architecture.md b/docs/01-data-qbit-architecture.md index 689568d..ea6f7c5 100644 --- a/docs/01-data-qbit-architecture.md +++ b/docs/01-data-qbit-architecture.md @@ -2,7 +2,7 @@ ## Table Prefixing -Data QBits support table prefixing for multi-instance deployment: +The producer can prefix metadata names. Prefixing alone does not guarantee multiple independent instances: review process names, relationships and any hard-coded table references before registering more than one instance. ```java public String applyPrefix(String tableName) @@ -54,6 +54,8 @@ public class ExampleEntity extends QRecordEntity private String code; // Natural key @QField - private Boolean isActive = true; // Soft delete + private Boolean isActive = true; // Used by the sync process's explicit deactivation logic } ``` + +An `isActive` field is not a global soft-delete facility. Add appropriate query filters and application behavior for inactive rows. diff --git a/docs/02-sync-process.md b/docs/02-sync-process.md index 20503f1..327c45c 100644 --- a/docs/02-sync-process.md +++ b/docs/02-sync-process.md @@ -27,44 +27,10 @@ Source Data (JSON) --> Sync Process --> Database Tables ## Implementation -```java -public class ExampleDataSyncStep extends AbstractTransformStep -{ - @Override - public void runOnePage(RunBackendStepInput input, RunBackendStepOutput output) - { - // 1. Load source data from classpath JSON - List sourceRecords = loadJsonData(resourcePath); +The checked implementation is [ExampleDataSyncStep](../src/main/java/com/kingsrook/qbits/example/sync/ExampleDataSyncStep.java), with process metadata in [ExampleDataSyncProcessMetaDataProducer](../src/main/java/com/kingsrook/qbits/example/sync/ExampleDataSyncProcessMetaDataProducer.java). It implements both `runOnePage` and the required `getProcessSummary` method. - // 2. Query existing by natural key - Map existingByKey = queryExisting(tableName, naturalKeyField); +The step loads classpath JSON, queries existing rows by their natural key, separates inserts/updates/deactivations, and calls its insert/update helpers. Deactivation explicitly sets `isActive=false` for removed active rows. Follow the source helper signatures when adapting the step, and review its execution behavior before introducing a preview or validation screen. The presence of an `isActive` field does not itself filter normal queries. - // 3. Categorize records - for(QRecord source : sourceRecords) - { - String key = source.getValueString(naturalKeyField); - QRecord existing = existingByKey.remove(key); - - if(existing == null) - toInsert.add(source); // New record - else if(hasChanges(source, existing)) - toUpdate.add(source); // Changed record - } - - // 4. Remaining in map - no longer in source - for(QRecord orphan : existingByKey.values()) - { - orphan.setValue("isActive", false); - toDeactivate.add(orphan); // Soft delete - } - - // 5. Execute operations - insertRecords(toInsert); - updateRecords(toUpdate); - updateRecords(toDeactivate); - } -} -``` ## Data File Format diff --git a/pom.xml b/pom.xml index 41d62c3..22ada99 100644 --- a/pom.xml +++ b/pom.xml @@ -17,7 +17,7 @@ 21 21 UTF-8 - 0.35.0 + 4.0.0 diff --git a/src/main/java/com/kingsrook/qbits/example/sync/ExampleDataSyncProcessMetaDataProducer.java b/src/main/java/com/kingsrook/qbits/example/sync/ExampleDataSyncProcessMetaDataProducer.java index 1a3184b..0d7be72 100644 --- a/src/main/java/com/kingsrook/qbits/example/sync/ExampleDataSyncProcessMetaDataProducer.java +++ b/src/main/java/com/kingsrook/qbits/example/sync/ExampleDataSyncProcessMetaDataProducer.java @@ -6,14 +6,14 @@ package com.kingsrook.qbits.example.sync; import java.util.List; import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.metadata.MetaDataProducerInterface; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionInputMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; -import com.kingsrook.qqq.backend.core.model.MetaDataProducerInterface; -import com.kingsrook.qbits.example.model.ExampleEntity; public class ExampleDataSyncProcessMetaDataProducer implements MetaDataProducerInterface @@ -31,15 +31,14 @@ public class ExampleDataSyncProcessMetaDataProducer implements MetaDataProducerI return new QProcessMetaData() .withName(NAME) .withLabel("Example Data Sync") - .withInputFields(List.of( - new QFieldMetaData("tableName", QFieldType.STRING), - new QFieldMetaData("naturalKeyField", QFieldType.STRING), - new QFieldMetaData("dataResourcePath", QFieldType.STRING) - )) .withStepList(List.of( new QBackendStepMetaData() .withName("sync") .withCode(new QCodeReference(ExampleDataSyncStep.class)) + .withInputData(new QFunctionInputMetaData() + .withField(new QFieldMetaData("tableName", QFieldType.STRING)) + .withField(new QFieldMetaData("naturalKeyField", QFieldType.STRING)) + .withField(new QFieldMetaData("dataResourcePath", QFieldType.STRING))) )); } } diff --git a/src/main/java/com/kingsrook/qbits/example/sync/ExampleDataSyncStep.java b/src/main/java/com/kingsrook/qbits/example/sync/ExampleDataSyncStep.java index e3a8217..477b91e 100644 --- a/src/main/java/com/kingsrook/qbits/example/sync/ExampleDataSyncStep.java +++ b/src/main/java/com/kingsrook/qbits/example/sync/ExampleDataSyncStep.java @@ -17,8 +17,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; -import com.kingsrook.qqq.backend.core.actions.processes.RunBackendStepInput; -import com.kingsrook.qqq.backend.core.actions.processes.RunBackendStepOutput; import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction; @@ -31,6 +29,7 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput; import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLineInterface; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractTransformStep; import org.json.JSONArray; @@ -236,4 +235,15 @@ public class ExampleDataSyncStep extends AbstractTransformStep updateInput.setRecords(records); new UpdateAction().execute(updateInput); } + + + + /******************************************************************************* + ** Return the process summary. + *******************************************************************************/ + @Override + public ArrayList getProcessSummary(RunBackendStepOutput runBackendStepOutput, boolean isForResultScreen) + { + return (new ArrayList<>()); + } }