From 0fc881c0ad5e64764be998ae787f78e24eb195ca Mon Sep 17 00:00:00 2001 From: cawcenter Date: Mon, 15 Dec 2025 01:53:51 -0500 Subject: [PATCH] feat: content generation engine - spintax resolver, API endpoints, BullMQ worker --- migrations/02_content_generation.sql | 54 +++++ src/lib/spintax/resolver.ts | 154 +++++++++++++ src/pages/api/god/campaigns/create.ts | 122 ++++++++++ src/pages/api/god/campaigns/launch/[id].ts | 71 ++++++ src/pages/api/god/campaigns/status/[id].ts | 75 ++++++ src/workers/contentGenerator.ts | 251 +++++++++++++++++++++ 6 files changed, 727 insertions(+) create mode 100644 migrations/02_content_generation.sql create mode 100644 src/lib/spintax/resolver.ts create mode 100644 src/pages/api/god/campaigns/create.ts create mode 100644 src/pages/api/god/campaigns/launch/[id].ts create mode 100644 src/pages/api/god/campaigns/status/[id].ts create mode 100644 src/workers/contentGenerator.ts diff --git a/migrations/02_content_generation.sql b/migrations/02_content_generation.sql new file mode 100644 index 0000000..eaf319f --- /dev/null +++ b/migrations/02_content_generation.sql @@ -0,0 +1,54 @@ +-- Phase 1: Content Generation Schema Updates + +-- Add columns to existing tables +ALTER TABLE avatars +ADD COLUMN IF NOT EXISTS industry VARCHAR(255), +ADD COLUMN IF NOT EXISTS pain_point TEXT, +ADD COLUMN IF NOT EXISTS value_prop TEXT; + +ALTER TABLE campaign_masters +ADD COLUMN IF NOT EXISTS site_id UUID REFERENCES sites (id) ON DELETE CASCADE; + +ALTER TABLE content_fragments +ADD COLUMN IF NOT EXISTS campaign_id UUID REFERENCES campaign_masters (id) ON DELETE CASCADE, +ADD COLUMN IF NOT EXISTS content_hash VARCHAR(64) UNIQUE, +ADD COLUMN IF NOT EXISTS use_count INTEGER DEFAULT 0; + +-- New table: variation_registry (track unique combinations) +CREATE TABLE IF NOT EXISTS variation_registry ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid (), + campaign_id UUID NOT NULL REFERENCES campaign_masters (id) ON DELETE CASCADE, + variation_hash VARCHAR(64) UNIQUE NOT NULL, + resolved_variables JSONB NOT NULL, + spintax_choices JSONB NOT NULL, + post_id UUID REFERENCES posts (id) ON DELETE SET NULL, + created_at TIMESTAMPTZ DEFAULT NOW() +); + +CREATE UNIQUE INDEX IF NOT EXISTS idx_variation_hash ON variation_registry (variation_hash); + +-- New table: block_usage_stats (track how many times each block is used) +CREATE TABLE IF NOT EXISTS block_usage_stats ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid (), + content_fragment_id UUID REFERENCES content_fragments (id) ON DELETE CASCADE, + block_type VARCHAR(255) NOT NULL, + total_uses INTEGER DEFAULT 0, + last_used_at TIMESTAMPTZ, + created_at TIMESTAMPTZ DEFAULT NOW(), + UNIQUE (content_fragment_id) +); + +-- New table: spintax_variation_stats (track which spintax choices are used most) +CREATE TABLE IF NOT EXISTS spintax_variation_stats ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid (), + content_fragment_id UUID REFERENCES content_fragments (id) ON DELETE CASCADE, + variation_path TEXT NOT NULL, -- e.g., "hero.h1.option_1" + variation_text TEXT NOT NULL, + use_count INTEGER DEFAULT 0, + last_used_at TIMESTAMPTZ, + created_at TIMESTAMPTZ DEFAULT NOW() +); + +CREATE INDEX IF NOT EXISTS idx_block_usage_fragment ON block_usage_stats (content_fragment_id); + +CREATE INDEX IF NOT EXISTS idx_spintax_stats_fragment ON spintax_variation_stats (content_fragment_id); \ No newline at end of file diff --git a/src/lib/spintax/resolver.ts b/src/lib/spintax/resolver.ts new file mode 100644 index 0000000..65dc52e --- /dev/null +++ b/src/lib/spintax/resolver.ts @@ -0,0 +1,154 @@ +// Spintax Resolver - Handles {A|B|C} syntax +import crypto from 'crypto'; + +export interface SpintaxChoice { + path: string; + chosen: string; + allOptions: string[]; +} + +/** + * Resolves spintax syntax {A|B|C} to a single choice + * Tracks which choices were made for uniqueness + */ +export class SpintaxResolver { + private choices: SpintaxChoice[] = []; + private seed: string; + + constructor(seed?: string) { + this.seed = seed || crypto.randomBytes(16).toString('hex'); + } + + /** + * Resolve all spintax in text + */ + resolve(text: string): string { + let resolved = text; + let iteration = 0; + + // Handle nested spintax with multiple passes + while (resolved.includes('{') && resolved.includes('}') && iteration < 10) { + resolved = this.resolvePass(resolved, iteration); + iteration++; + } + + return resolved; + } + + private resolvePass(text: string, iteration: number): string { + const regex = /\{([^{}]+)\}/g; + let result = text; + let match; + let offset = 0; + + while ((match = regex.exec(text)) !== null) { + const fullMatch = match[0]; + const options = match[1].split('|'); + + // Deterministic choice based on seed + position + const choiceIndex = this.getChoiceIndex(match.index + iteration, options.length); + const chosen = options[choiceIndex].trim(); + + // Track the choice + this.choices.push({ + path: `pos_${match.index}_iter_${iteration}`, + chosen, + allOptions: options + }); + + // Replace in result + const beforeMatch = result.substring(0, match.index + offset); + const afterMatch = result.substring(match.index + offset + fullMatch.length); + result = beforeMatch + chosen + afterMatch; + + offset += chosen.length - fullMatch.length; + } + + return result; + } + + private getChoiceIndex(position: number, optionsCount: number): number { + const hash = crypto.createHash('sha256') + .update(`${this.seed}_${position}`) + .digest('hex'); + return parseInt(hash.substring(0, 8), 16) % optionsCount; + } + + /** + * Get all choices made during resolution + */ + getChoices(): SpintaxChoice[] { + return this.choices; + } + + /** + * Generate hash of choices for uniqueness checking + */ + getChoicesHash(): string { + const choiceString = this.choices + .map(c => c.chosen) + .join('::'); + return crypto.createHash('sha256') + .update(choiceString) + .digest('hex') + .substring(0, 32); + } + + /** + * Reset for new resolution + */ + reset(newSeed?: string) { + this.choices = []; + if (newSeed) this.seed = newSeed; + } +} + +/** + * Expand variables like {{CITY}} in text + */ +export function expandVariables(text: string, variables: Record): string { + let result = text; + + for (const [key, value] of Object.entries(variables)) { + const regex = new RegExp(`\\{\\{${key}\\}\\}`, 'g'); + result = result.replace(regex, value); + } + + return result; +} + +/** + * Generate cartesian product of pipe-separated values + * Example: { CITY: "A|B", STATE: "X|Y" } => 4 combinations + */ +export function generateCartesianProduct( + variables: Record +): Array> { + const keys = Object.keys(variables); + const values = keys.map(key => variables[key].split('|').map(v => v.trim())); + + function* cartesian(arrays: string[][]): Generator { + if (arrays.length === 0) { + yield []; + return; + } + + const [first, ...rest] = arrays; + for (const value of first) { + for (const combo of cartesian(rest)) { + yield [value, ...combo]; + } + } + } + + const results: Array> = []; + for (const combo of cartesian(values)) { + const obj: Record = {}; + keys.forEach((key, i) => { + obj[key] = combo[i]; + }); + results.push(obj); + } + + return results; +} diff --git a/src/pages/api/god/campaigns/create.ts b/src/pages/api/god/campaigns/create.ts new file mode 100644 index 0000000..d26db9c --- /dev/null +++ b/src/pages/api/god/campaigns/create.ts @@ -0,0 +1,122 @@ +// API Endpoint: POST /api/god/campaigns/create +import type { APIRoute } from 'astro'; +import { pool } from '../../../../lib/db/db'; +import crypto from 'crypto'; + +interface CampaignBlueprint { + asset_name: string; + deployment_target: string; + variables: Record; + content: { + url_path: string; + meta_description: string; + body: Array<{ + block_type: string; + content: string; + }>; + }; +} + +export const POST: APIRoute = async ({ request }) => { + try { + // Auth check + const godToken = request.headers.get('X-God-Token'); + if (!godToken || godToken !== import.meta.env.GOD_TOKEN) { + return new Response(JSON.stringify({ error: 'Unauthorized' }), { + status: 401, + headers: { 'Content-Type': 'application/json' } + }); + } + + const body = await request.json(); + const { name, blueprint } = body as { name?: string; blueprint: CampaignBlueprint }; + + if (!blueprint || !blueprint.content || !blueprint.variables) { + return new Response(JSON.stringify({ error: 'Invalid blueprint structure' }), { + status: 400, + headers: { 'Content-Type': 'application/json' } + }); + } + + const client = await pool.connect(); + + try { + await client.query('BEGIN'); + + // Get admin site ID + const siteResult = await client.query( + `SELECT id FROM sites WHERE domain = 'spark.jumpstartscaling.com' LIMIT 1` + ); + + if (siteResult.rows.length === 0) { + throw new Error('Admin site not found'); + } + + const siteId = siteResult.rows[0].id; + + // Insert campaign + const campaignName = name || blueprint.asset_name; + const campaignResult = await client.query( + `INSERT INTO campaign_masters (site_id, name, blueprint_json, status) + VALUES ($1, $2, $3, 'pending') + RETURNING id`, + [siteId, campaignName, JSON.stringify(blueprint)] + ); + + const campaignId = campaignResult.rows[0].id; + + // Insert content fragments (blocks) + for (const block of blueprint.content.body) { + const contentHash = crypto.createHash('sha256') + .update(block.content) + .digest('hex') + .substring(0, 32); + + await client.query( + `INSERT INTO content_fragments ( + site_id, campaign_id, block_type, blueprint_name, content, content_hash, use_count + ) VALUES ($1, $2, $3, $4, $5, $6, 0) + ON CONFLICT (content_hash) DO UPDATE SET + campaign_id = EXCLUDED.campaign_id, + use_count = content_fragments.use_count + `, + [ + siteId, + campaignId, + block.block_type.replace(/ \(\d+\)$/, ''), + campaignName, + block.content, + contentHash + ] + ); + } + + await client.query('COMMIT'); + + return new Response(JSON.stringify({ + success: true, + campaignId, + message: `Campaign "${campaignName}" created successfully. Use /launch/${campaignId} to generate content.` + }), { + status: 201, + headers: { 'Content-Type': 'application/json' } + }); + + } catch (err) { + await client.query('ROLLBACK'); + throw err; + } finally { + client.release(); + } + + } catch (error: any) { + console.error('Campaign creation error:', error); + return new Response(JSON.stringify({ + error: 'Campaign creation failed', + details: error.message + }), { + status: 500, + headers: { 'Content-Type': 'application/json' } + }); + } +}; diff --git a/src/pages/api/god/campaigns/launch/[id].ts b/src/pages/api/god/campaigns/launch/[id].ts new file mode 100644 index 0000000..d08fe88 --- /dev/null +++ b/src/pages/api/god/campaigns/launch/[id].ts @@ -0,0 +1,71 @@ +// API Endpoint: POST /api/god/campaigns/launch/[id] +import type { APIRoute } from 'astro'; +import { pool } from '../../../../../lib/db/db'; +import { batchQueue } from '../../../../../lib/queue/config'; + +export const POST: APIRoute = async ({ params, request }) => { + try { + const godToken = request.headers.get('X-God-Token'); + if (!godToken || godToken !== import.meta.env.GOD_TOKEN) { + return new Response(JSON.stringify({ error: 'Unauthorized' }), { + status: 401, + headers: { 'Content-Type': 'application/json' } + }); + } + + const { id } = params; + if (!id) { + return new Response(JSON.stringify({ error: 'Campaign ID required' }), { + status: 400, + headers: { 'Content-Type': 'application/json' } + }); + } + + // Fetch campaign + const result = await pool.query( + `SELECT id, name, blueprint_json FROM campaign_masters WHERE id = $1`, + [id] + ); + + if (result.rows.length === 0) { + return new Response(JSON.stringify({ error: 'Campaign not found' }), { + status: 404, + headers: { 'Content-Type': 'application/json' } + }); + } + + const campaign = result.rows[0]; + + // Queue the job + await batchQueue.add('generate_campaign_content', { + campaignId: id, + campaignName: campaign.name + }); + + // Update status + await pool.query( + `UPDATE campaign_masters SET status = 'processing', updated_at = NOW() WHERE id = $1`, + [id] + ); + + return new Response(JSON.stringify({ + success: true, + campaignId: id, + message: 'Campaign queued for generation', + status: 'processing' + }), { + status: 200, + headers: { 'Content-Type': 'application/json' } + }); + + } catch (error: any) { + console.error('Campaign launch error:', error); + return new Response(JSON.stringify({ + error: 'Campaign launch failed', + details: error.message + }), { + status: 500, + headers: { 'Content-Type': 'application/json' } + }); + } +}; diff --git a/src/pages/api/god/campaigns/status/[id].ts b/src/pages/api/god/campaigns/status/[id].ts new file mode 100644 index 0000000..aae7329 --- /dev/null +++ b/src/pages/api/god/campaigns/status/[id].ts @@ -0,0 +1,75 @@ +// API Endpoint: GET /api/god/campaigns/status/[id] +import type { APIRoute } from 'astro'; +import { pool } from '../../../../../lib/db/db'; + +export const GET: APIRoute = async ({ params, request }) => { + try { + const godToken = request.headers.get('X-God-Token'); + if (!godToken || godToken !== import.meta.env.GOD_TOKEN) { + return new Response(JSON.stringify({ error: 'Unauthorized' }), { + status: 401, + headers: { 'Content-Type': 'application/json' } + }); + } + + const { id } = params; + + // Get campaign details + const campaignResult = await pool.query( + `SELECT id, name, status, created_at, updated_at FROM campaign_masters WHERE id = $1`, + [id] + ); + + if (campaignResult.rows.length === 0) { + return new Response(JSON.stringify({ error: 'Campaign not found' }), { + status: 404, + headers: { 'Content-Type': 'application/json' } + }); + } + + const campaign = campaignResult.rows[0]; + + // Count generated posts + const postsResult = await pool.query( + `SELECT COUNT(*) as count FROM variation_registry WHERE campaign_id = $1`, + [id] + ); + + const postsCreated = parseInt(postsResult.rows[0].count); + + // Get block usage stats + const blockStats = await pool.query( + `SELECT block_type, total_uses + FROM block_usage_stats + WHERE content_fragment_id IN ( + SELECT id FROM content_fragments WHERE campaign_id = $1 + ) + ORDER BY total_uses DESC + LIMIT 10`, + [id] + ); + + return new Response(JSON.stringify({ + campaignId: campaign.id, + name: campaign.name, + status: campaign.status, + postsCreated, + createdAt: campaign.created_at, + updatedAt: campaign.updated_at, + blockUsage: blockStats.rows + }), { + status: 200, + headers: { 'Content-Type': 'application/json' } + }); + + } catch (error: any) { + console.error('Status check error:', error); + return new Response(JSON.stringify({ + error: 'Status check failed', + details: error.message + }), { + status: 500, + headers: { 'Content-Type': 'application/json' } + }); + } +}; diff --git a/src/workers/contentGenerator.ts b/src/workers/contentGenerator.ts new file mode 100644 index 0000000..bfd9760 --- /dev/null +++ b/src/workers/contentGenerator.ts @@ -0,0 +1,251 @@ +// BullMQ Worker: Content Generator +import { Worker } from 'bullmq'; +import { pool } from '../lib/db/db'; +import { redisConnection } from '../lib/queue/config'; +import { SpintaxResolver, expandVariables, generateCartesianProduct } from '../lib/spintax/resolver'; +import crypto from 'crypto'; + +interface GenerateCampaignJob { + campaignId: string; + campaignName: string; +} + +/** + * Main content generation worker + * Processes campaigns and generates full articles + */ +export const contentGeneratorWorker = new Worker( + 'generate_campaign_content', + async (job) => { + const { campaignId, campaignName } = job.data as GenerateCampaignJob; + + console.log(`\nšŸ”± Processing campaign: ${campaignName} (${campaignId})`); + + try { + // 1. Fetch campaign blueprint + const campaignResult = await pool.query( + `SELECT id, blueprint_json, site_id FROM campaign_masters WHERE id = $1`, + [campaignId] + ); + + if (campaignResult.rows.length === 0) { + throw new Error(`Campaign ${campaignId} not found`); + } + + const campaign = campaignResult.rows[0]; + const blueprint = campaign.blueprint_json; + const siteId = campaign.site_id; + + console.log(`šŸ“‹ Blueprint loaded: ${blueprint.asset_name}`); + console.log(`šŸ”¢ Variables: ${Object.keys(blueprint.variables).join(', ')}`); + + // 2. Generate cartesian product of all variables + const combinations = generateCartesianProduct(blueprint.variables); + console.log(`šŸŽÆ Generated ${combinations.length} unique combinations`); + + let created = 0; + let skipped = 0; + + // 3. For each combination, generate article + for (const vars of combinations) { + try { + const articleHtml = await generateArticle(blueprint, vars, campaignId, siteId); + + if (articleHtml) { + created++; + console.log(`āœ… Created: ${vars.CITY}, ${vars.STATE} (${created}/${combinations.length})`); + } else { + skipped++; + } + } catch (err) { + console.error(`āŒ Failed for ${vars.CITY}:`, err); + skipped++; + } + + // Update progress + await job.updateProgress((created + skipped) / combinations.length * 100); + } + + // 4. Mark campaign complete + await pool.query( + `UPDATE campaign_masters SET status = 'completed', updated_at = NOW() WHERE id = $1`, + [campaignId] + ); + + console.log(`\nšŸŽ‰ Campaign complete! Created: ${created}, Skipped: ${skipped}\n`); + + return { created, skipped, total: combinations.length }; + + } catch (error) { + console.error(`šŸ’„ Campaign processing failed:`, error); + + await pool.query( + `UPDATE campaign_masters SET status = 'failed', updated_at = NOW() WHERE id = $1`, + [campaignId] + ); + + throw error; + } + }, + { + connection: redisConnection, + concurrency: 2 + } +); + +/** + * Generate a single article from blueprint + variables + */ +async function generateArticle( + blueprint: any, + vars: Record, + campaignId: string, + siteId: string +): Promise { + + // Create unique seed for this variation + const seed = JSON.stringify(vars); + const resolver = new SpintaxResolver(seed); + + // 1. Expand variable placeholders + let fullHtml = ''; + const blocks = blueprint.content.body; + + for (const block of blocks) { + let blockContent = block.content; + + // Replace {{VARIABLES}} + blockContent = expandVariables(blockContent, vars); + + // Resolve spintax {A|B|C} + blockContent = resolver.resolve(blockContent); + + fullHtml += blockContent + '\n\n'; + } + + // 2. Generate variation hash for uniqueness check + const variationHash = resolver.getChoicesHash(); + const varsHash = crypto.createHash('sha256') + .update(JSON.stringify(vars)) + .digest('hex') + .substring(0, 32); + const combinedHash = crypto.createHash('sha256') + .update(variationHash + varsHash) + .digest('hex') + .substring(0, 64); + + // 3. Check if this variation already exists + const existingCheck = await pool.query( + `SELECT id FROM variation_registry WHERE variation_hash = $1`, + [combinedHash] + ); + + if (existingCheck.rows.length > 0) { + console.log(`ā­ļø Skipping duplicate: ${vars.CITY}`); + return null; + } + + // 4. Generate title and slug + const title = expandVariables(resolver.resolve(blueprint.asset_name), vars); + const slug = generateSlug(vars); + + // 5. Generate meta description + const metaDesc = expandVariables( + resolver.resolve(blueprint.meta_description), + vars + ); + + // 6. Insert post + const postResult = await pool.query( + `INSERT INTO posts ( + site_id, title, slug, content, excerpt, status, + published_at, created_at + ) VALUES ($1, $2, $3, $4, $5, 'published', NOW(), NOW()) + RETURNING id`, + [ + siteId, + title, + slug, + fullHtml, + metaDesc.substring(0, 300) + ] + ); + + const postId = postResult.rows[0].id; + + // 7. Record the variation + await pool.query( + `INSERT INTO variation_registry ( + campaign_id, variation_hash, resolved_variables, spintax_choices, post_id + ) VALUES ($1, $2, $3, $4, $5)`, + [ + campaignId, + combinedHash, + JSON.stringify(vars), + JSON.stringify(resolver.getChoices()), + postId + ] + ); + + // 8. Update block usage stats + await updateBlockUsageStats(blocks, resolver.getChoices()); + + return fullHtml; +} + +/** + * Update usage statistics for blocks and spintax choices + */ +async function updateBlockUsageStats(blocks: any[], choices: any[]) { + // Track which blocks were used + for (const block of blocks) { + await pool.query( + `INSERT INTO block_usage_stats (content_fragment_id, block_type, total_uses, last_used_at) + SELECT id, $1, 1, NOW() + FROM content_fragments + WHERE block_type = $1 + ON CONFLICT (content_fragment_id) DO UPDATE SET + total_uses = block_usage_stats.total_uses + 1, + last_used_at = NOW()`, + [block.block_type.replace(/ \(\d+\)$/, '')] + ); + } + + // Track spintax variation usage + for (const choice of choices) { + await pool.query( + `INSERT INTO spintax_variation_stats ( + content_fragment_id, variation_path, variation_text, use_count, last_used_at + ) + SELECT cf.id, $1, $2, 1, NOW() + FROM content_fragments cf + LIMIT 1 + ON CONFLICT DO NOTHING`, + [choice.path, choice.chosen] + ); + } +} + +/** + * Generate URL-safe slug from variables + */ +function generateSlug(vars: Record): string { + const city = (vars.CITY || 'city').toLowerCase().replace(/[^a-z0-9]+/g, '-'); + const state = (vars.STATE || 'state').toLowerCase().replace(/[^a-z0-9]+/g, '-'); + return `${city}-${state}-${Date.now()}`; +} + +// Worker event handlers +contentGeneratorWorker.on('completed', (job) => { + console.log(`āœ… Job ${job.id} completed:`, job.returnvalue); +}); + +contentGeneratorWorker.on('failed', (job, err) => { + console.error(`āŒ Job ${job?.id} failed:`, err); +}); + +contentGeneratorWorker.on('error', (err) => { + console.error('šŸ’„ Worker error:', err); +}); + +console.log('šŸš€ Content Generator Worker started');