feat: content generation engine - spintax resolver, API endpoints, BullMQ worker

migrations/02_content_generation.sql

@@ -0,0 +1,54 @@
+-- Phase 1: Content Generation Schema Updates
+
+-- Add columns to existing tables
+ALTER TABLE avatars
+ADD COLUMN IF NOT EXISTS industry VARCHAR(255),
+ADD COLUMN IF NOT EXISTS pain_point TEXT,
+ADD COLUMN IF NOT EXISTS value_prop TEXT;
+
+ALTER TABLE campaign_masters
+ADD COLUMN IF NOT EXISTS site_id UUID REFERENCES sites (id) ON DELETE CASCADE;
+
+ALTER TABLE content_fragments
+ADD COLUMN IF NOT EXISTS campaign_id UUID REFERENCES campaign_masters (id) ON DELETE CASCADE,
+ADD COLUMN IF NOT EXISTS content_hash VARCHAR(64) UNIQUE,
+ADD COLUMN IF NOT EXISTS use_count INTEGER DEFAULT 0;
+
+-- New table: variation_registry (track unique combinations)
+CREATE TABLE IF NOT EXISTS variation_registry (
+    id UUID PRIMARY KEY DEFAULT gen_random_uuid (),
+    campaign_id UUID NOT NULL REFERENCES campaign_masters (id) ON DELETE CASCADE,
+    variation_hash VARCHAR(64) UNIQUE NOT NULL,
+    resolved_variables JSONB NOT NULL,
+    spintax_choices JSONB NOT NULL,
+    post_id UUID REFERENCES posts (id) ON DELETE SET NULL,
+    created_at TIMESTAMPTZ DEFAULT NOW()
+);
+
+CREATE UNIQUE INDEX IF NOT EXISTS idx_variation_hash ON variation_registry (variation_hash);
+
+-- New table: block_usage_stats (track how many times each block is used)
+CREATE TABLE IF NOT EXISTS block_usage_stats (
+    id UUID PRIMARY KEY DEFAULT gen_random_uuid (),
+    content_fragment_id UUID REFERENCES content_fragments (id) ON DELETE CASCADE,
+    block_type VARCHAR(255) NOT NULL,
+    total_uses INTEGER DEFAULT 0,
+    last_used_at TIMESTAMPTZ,
+    created_at TIMESTAMPTZ DEFAULT NOW(),
+    UNIQUE (content_fragment_id)
+);
+
+-- New table: spintax_variation_stats (track which spintax choices are used most)
+CREATE TABLE IF NOT EXISTS spintax_variation_stats (
+    id UUID PRIMARY KEY DEFAULT gen_random_uuid (),
+    content_fragment_id UUID REFERENCES content_fragments (id) ON DELETE CASCADE,
+    variation_path TEXT NOT NULL, -- e.g., "hero.h1.option_1"
+    variation_text TEXT NOT NULL,
+    use_count INTEGER DEFAULT 0,
+    last_used_at TIMESTAMPTZ,
+    created_at TIMESTAMPTZ DEFAULT NOW()
+);
+
+CREATE INDEX IF NOT EXISTS idx_block_usage_fragment ON block_usage_stats (content_fragment_id);
+
+CREATE INDEX IF NOT EXISTS idx_spintax_stats_fragment ON spintax_variation_stats (content_fragment_id);

src/lib/spintax/resolver.ts

@@ -0,0 +1,154 @@
+// Spintax Resolver - Handles {A|B|C} syntax
+import crypto from 'crypto';
+
+export interface SpintaxChoice {
+    path: string;
+    chosen: string;
+    allOptions: string[];
+}
+
+/**
+ * Resolves spintax syntax {A|B|C} to a single choice
+ * Tracks which choices were made for uniqueness
+ */
+export class SpintaxResolver {
+    private choices: SpintaxChoice[] = [];
+    private seed: string;
+
+    constructor(seed?: string) {
+        this.seed = seed || crypto.randomBytes(16).toString('hex');
+    }
+
+    /**
+     * Resolve all spintax in text
+     */
+    resolve(text: string): string {
+        let resolved = text;
+        let iteration = 0;
+
+        // Handle nested spintax with multiple passes
+        while (resolved.includes('{') && resolved.includes('}') && iteration < 10) {
+            resolved = this.resolvePass(resolved, iteration);
+            iteration++;
+        }
+
+        return resolved;
+    }
+
+    private resolvePass(text: string, iteration: number): string {
+        const regex = /\{([^{}]+)\}/g;
+        let result = text;
+        let match;
+        let offset = 0;
+
+        while ((match = regex.exec(text)) !== null) {
+            const fullMatch = match[0];
+            const options = match[1].split('|');
+
+            // Deterministic choice based on seed + position
+            const choiceIndex = this.getChoiceIndex(match.index + iteration, options.length);
+            const chosen = options[choiceIndex].trim();
+
+            // Track the choice
+            this.choices.push({
+                path: `pos_${match.index}_iter_${iteration}`,
+                chosen,
+                allOptions: options
+            });
+
+            // Replace in result
+            const beforeMatch = result.substring(0, match.index + offset);
+            const afterMatch = result.substring(match.index + offset + fullMatch.length);
+            result = beforeMatch + chosen + afterMatch;
+
+            offset += chosen.length - fullMatch.length;
+        }
+
+        return result;
+    }
+
+    private getChoiceIndex(position: number, optionsCount: number): number {
+        const hash = crypto.createHash('sha256')
+            .update(`${this.seed}_${position}`)
+            .digest('hex');
+        return parseInt(hash.substring(0, 8), 16) % optionsCount;
+    }
+
+    /**
+     * Get all choices made during resolution
+     */
+    getChoices(): SpintaxChoice[] {
+        return this.choices;
+    }
+
+    /**
+     * Generate hash of choices for uniqueness checking
+     */
+    getChoicesHash(): string {
+        const choiceString = this.choices
+            .map(c => c.chosen)
+            .join('::');
+        return crypto.createHash('sha256')
+            .update(choiceString)
+            .digest('hex')
+            .substring(0, 32);
+    }
+
+    /**
+     * Reset for new resolution
+     */
+    reset(newSeed?: string) {
+        this.choices = [];
+        if (newSeed) this.seed = newSeed;
+    }
+}
+
+/**
+ * Expand variables like {{CITY}} in text
+ */
+export function expandVariables(text: string, variables: Record<string, string>): string {
+    let result = text;
+
+    for (const [key, value] of Object.entries(variables)) {
+        const regex = new RegExp(`\\{\\{${key}\\}\\}`, 'g');
+        result = result.replace(regex, value);
+    }
+
+    return result;
+}
+
+/**
+ * Generate cartesian product of pipe-separated values
+ * Example: { CITY: "A|B", STATE: "X|Y" } => 4 combinations
+ */
+export function generateCartesianProduct(
+    variables: Record<string, string>
+): Array<Record<string, string>> {
+    const keys = Object.keys(variables);
+    const values = keys.map(key => variables[key].split('|').map(v => v.trim()));
+
+    function* cartesian(arrays: string[][]): Generator<string[]> {
+        if (arrays.length === 0) {
+            yield [];
+            return;
+        }
+
+        const [first, ...rest] = arrays;
+        for (const value of first) {
+            for (const combo of cartesian(rest)) {
+                yield [value, ...combo];
+            }
+        }
+    }
+
+    const results: Array<Record<string, string>> = [];
+    for (const combo of cartesian(values)) {
+        const obj: Record<string, string> = {};
+        keys.forEach((key, i) => {
+            obj[key] = combo[i];
+        });
+        results.push(obj);
+    }
+
+    return results;
+}

src/pages/api/god/campaigns/create.ts

@@ -0,0 +1,122 @@
+// API Endpoint: POST /api/god/campaigns/create
+import type { APIRoute } from 'astro';
+import { pool } from '../../../../lib/db/db';
+import crypto from 'crypto';
+
+interface CampaignBlueprint {
+    asset_name: string;
+    deployment_target: string;
+    variables: Record<string, string>;
+    content: {
+        url_path: string;
+        meta_description: string;
+        body: Array<{
+            block_type: string;
+            content: string;
+        }>;
+    };
+}
+
+export const POST: APIRoute = async ({ request }) => {
+    try {
+        // Auth check
+        const godToken = request.headers.get('X-God-Token');
+        if (!godToken || godToken !== import.meta.env.GOD_TOKEN) {
+            return new Response(JSON.stringify({ error: 'Unauthorized' }), {
+                status: 401,
+                headers: { 'Content-Type': 'application/json' }
+            });
+        }
+
+        const body = await request.json();
+        const { name, blueprint } = body as { name?: string; blueprint: CampaignBlueprint };
+
+        if (!blueprint || !blueprint.content || !blueprint.variables) {
+            return new Response(JSON.stringify({ error: 'Invalid blueprint structure' }), {
+                status: 400,
+                headers: { 'Content-Type': 'application/json' }
+            });
+        }
+
+        const client = await pool.connect();
+
+        try {
+            await client.query('BEGIN');
+
+            // Get admin site ID
+            const siteResult = await client.query(
+                `SELECT id FROM sites WHERE domain = 'spark.jumpstartscaling.com' LIMIT 1`
+            );
+
+            if (siteResult.rows.length === 0) {
+                throw new Error('Admin site not found');
+            }
+
+            const siteId = siteResult.rows[0].id;
+
+            // Insert campaign
+            const campaignName = name || blueprint.asset_name;
+            const campaignResult = await client.query(
+                `INSERT INTO campaign_masters (site_id, name, blueprint_json, status)
+         VALUES ($1, $2, $3, 'pending')
+         RETURNING id`,
+                [siteId, campaignName, JSON.stringify(blueprint)]
+            );
+
+            const campaignId = campaignResult.rows[0].id;
+
+            // Insert content fragments (blocks)
+            for (const block of blueprint.content.body) {
+                const contentHash = crypto.createHash('sha256')
+                    .update(block.content)
+                    .digest('hex')
+                    .substring(0, 32);
+
+                await client.query(
+                    `INSERT INTO content_fragments (
+            site_id, campaign_id, block_type, blueprint_name, content, content_hash, use_count
+          ) VALUES ($1, $2, $3, $4, $5, $6, 0)
+          ON CONFLICT (content_hash) DO UPDATE SET
+            campaign_id = EXCLUDED.campaign_id,
+            use_count = content_fragments.use_count
+          `,
+                    [
+                        siteId,
+                        campaignId,
+                        block.block_type.replace(/ \(\d+\)$/, ''),
+                        campaignName,
+                        block.content,
+                        contentHash
+                    ]
+                );
+            }
+
+            await client.query('COMMIT');
+
+            return new Response(JSON.stringify({
+                success: true,
+                campaignId,
+                message: `Campaign "${campaignName}" created successfully. Use /launch/${campaignId} to generate content.`
+            }), {
+                status: 201,
+                headers: { 'Content-Type': 'application/json' }
+            });
+
+        } catch (err) {
+            await client.query('ROLLBACK');
+            throw err;
+        } finally {
+            client.release();
+        }
+
+    } catch (error: any) {
+        console.error('Campaign creation error:', error);
+        return new Response(JSON.stringify({
+            error: 'Campaign creation failed',
+            details: error.message
+        }), {
+            status: 500,
+            headers: { 'Content-Type': 'application/json' }
+        });
+    }
+};

src/pages/api/god/campaigns/launch/[id].ts

@@ -0,0 +1,71 @@
+// API Endpoint: POST /api/god/campaigns/launch/[id]
+import type { APIRoute } from 'astro';
+import { pool } from '../../../../../lib/db/db';
+import { batchQueue } from '../../../../../lib/queue/config';
+
+export const POST: APIRoute = async ({ params, request }) => {
+    try {
+        const godToken = request.headers.get('X-God-Token');
+        if (!godToken || godToken !== import.meta.env.GOD_TOKEN) {
+            return new Response(JSON.stringify({ error: 'Unauthorized' }), {
+                status: 401,
+                headers: { 'Content-Type': 'application/json' }
+            });
+        }
+
+        const { id } = params;
+        if (!id) {
+            return new Response(JSON.stringify({ error: 'Campaign ID required' }), {
+                status: 400,
+                headers: { 'Content-Type': 'application/json' }
+            });
+        }
+
+        // Fetch campaign
+        const result = await pool.query(
+            `SELECT id, name, blueprint_json FROM campaign_masters WHERE id = $1`,
+            [id]
+        );
+
+        if (result.rows.length === 0) {
+            return new Response(JSON.stringify({ error: 'Campaign not found' }), {
+                status: 404,
+                headers: { 'Content-Type': 'application/json' }
+            });
+        }
+
+        const campaign = result.rows[0];
+
+        // Queue the job
+        await batchQueue.add('generate_campaign_content', {
+            campaignId: id,
+            campaignName: campaign.name
+        });
+
+        // Update status
+        await pool.query(
+            `UPDATE campaign_masters SET status = 'processing', updated_at = NOW() WHERE id = $1`,
+            [id]
+        );
+
+        return new Response(JSON.stringify({
+            success: true,
+            campaignId: id,
+            message: 'Campaign queued for generation',
+            status: 'processing'
+        }), {
+            status: 200,
+            headers: { 'Content-Type': 'application/json' }
+        });
+
+    } catch (error: any) {
+        console.error('Campaign launch error:', error);
+        return new Response(JSON.stringify({
+            error: 'Campaign launch failed',
+            details: error.message
+        }), {
+            status: 500,
+            headers: { 'Content-Type': 'application/json' }
+        });
+    }
+};

src/pages/api/god/campaigns/status/[id].ts

@@ -0,0 +1,75 @@
+// API Endpoint: GET /api/god/campaigns/status/[id]
+import type { APIRoute } from 'astro';
+import { pool } from '../../../../../lib/db/db';
+
+export const GET: APIRoute = async ({ params, request }) => {
+    try {
+        const godToken = request.headers.get('X-God-Token');
+        if (!godToken || godToken !== import.meta.env.GOD_TOKEN) {
+            return new Response(JSON.stringify({ error: 'Unauthorized' }), {
+                status: 401,
+                headers: { 'Content-Type': 'application/json' }
+            });
+        }
+
+        const { id } = params;
+
+        // Get campaign details
+        const campaignResult = await pool.query(
+            `SELECT id, name, status, created_at, updated_at FROM campaign_masters WHERE id = $1`,
+            [id]
+        );
+
+        if (campaignResult.rows.length === 0) {
+            return new Response(JSON.stringify({ error: 'Campaign not found' }), {
+                status: 404,
+                headers: { 'Content-Type': 'application/json' }
+            });
+        }
+
+        const campaign = campaignResult.rows[0];
+
+        // Count generated posts
+        const postsResult = await pool.query(
+            `SELECT COUNT(*) as count FROM variation_registry WHERE campaign_id = $1`,
+            [id]
+        );
+
+        const postsCreated = parseInt(postsResult.rows[0].count);
+
+        // Get block usage stats
+        const blockStats = await pool.query(
+            `SELECT block_type, total_uses 
+       FROM block_usage_stats 
+       WHERE content_fragment_id IN (
+         SELECT id FROM content_fragments WHERE campaign_id = $1
+       )
+       ORDER BY total_uses DESC
+       LIMIT 10`,
+            [id]
+        );
+
+        return new Response(JSON.stringify({
+            campaignId: campaign.id,
+            name: campaign.name,
+            status: campaign.status,
+            postsCreated,
+            createdAt: campaign.created_at,
+            updatedAt: campaign.updated_at,
+            blockUsage: blockStats.rows
+        }), {
+            status: 200,
+            headers: { 'Content-Type': 'application/json' }
+        });
+
+    } catch (error: any) {
+        console.error('Status check error:', error);
+        return new Response(JSON.stringify({
+            error: 'Status check failed',
+            details: error.message
+        }), {
+            status: 500,
+            headers: { 'Content-Type': 'application/json' }
+        });
+    }
+};

src/workers/contentGenerator.ts

@@ -0,0 +1,251 @@
+// BullMQ Worker: Content Generator
+import { Worker } from 'bullmq';
+import { pool } from '../lib/db/db';
+import { redisConnection } from '../lib/queue/config';
+import { SpintaxResolver, expandVariables, generateCartesianProduct } from '../lib/spintax/resolver';
+import crypto from 'crypto';
+
+interface GenerateCampaignJob {
+    campaignId: string;
+    campaignName: string;
+}
+
+/**
+ * Main content generation worker
+ * Processes campaigns and generates full articles
+ */
+export const contentGeneratorWorker = new Worker(
+    'generate_campaign_content',
+    async (job) => {
+        const { campaignId, campaignName } = job.data as GenerateCampaignJob;
+
+        console.log(`\n🔱 Processing campaign: ${campaignName} (${campaignId})`);
+
+        try {
+            // 1. Fetch campaign blueprint
+            const campaignResult = await pool.query(
+                `SELECT id, blueprint_json, site_id FROM campaign_masters WHERE id = $1`,
+                [campaignId]
+            );
+
+            if (campaignResult.rows.length === 0) {
+                throw new Error(`Campaign ${campaignId} not found`);
+            }
+
+            const campaign = campaignResult.rows[0];
+            const blueprint = campaign.blueprint_json;
+            const siteId = campaign.site_id;
+
+            console.log(`📋 Blueprint loaded: ${blueprint.asset_name}`);
+            console.log(`🔢 Variables: ${Object.keys(blueprint.variables).join(', ')}`);
+
+            // 2. Generate cartesian product of all variables
+            const combinations = generateCartesianProduct(blueprint.variables);
+            console.log(`🎯 Generated ${combinations.length} unique combinations`);
+
+            let created = 0;
+            let skipped = 0;
+
+            // 3. For each combination, generate article
+            for (const vars of combinations) {
+                try {
+                    const articleHtml = await generateArticle(blueprint, vars, campaignId, siteId);
+
+                    if (articleHtml) {
+                        created++;
+                        console.log(`✅ Created: ${vars.CITY}, ${vars.STATE} (${created}/${combinations.length})`);
+                    } else {
+                        skipped++;
+                    }
+                } catch (err) {
+                    console.error(`❌ Failed for ${vars.CITY}:`, err);
+                    skipped++;
+                }
+
+                // Update progress
+                await job.updateProgress((created + skipped) / combinations.length * 100);
+            }
+
+            // 4. Mark campaign complete
+            await pool.query(
+                `UPDATE campaign_masters SET status = 'completed', updated_at = NOW() WHERE id = $1`,
+                [campaignId]
+            );
+
+            console.log(`\n🎉 Campaign complete! Created: ${created}, Skipped: ${skipped}\n`);
+
+            return { created, skipped, total: combinations.length };
+
+        } catch (error) {
+            console.error(`💥 Campaign processing failed:`, error);
+
+            await pool.query(
+                `UPDATE campaign_masters SET status = 'failed', updated_at = NOW() WHERE id = $1`,
+                [campaignId]
+            );
+
+            throw error;
+        }
+    },
+    {
+        connection: redisConnection,
+        concurrency: 2
+    }
+);
+
+/**
+ * Generate a single article from blueprint + variables
+ */
+async function generateArticle(
+    blueprint: any,
+    vars: Record<string, string>,
+    campaignId: string,
+    siteId: string
+): Promise<string | null> {
+
+    // Create unique seed for this variation
+    const seed = JSON.stringify(vars);
+    const resolver = new SpintaxResolver(seed);
+
+    // 1. Expand variable placeholders
+    let fullHtml = '';
+    const blocks = blueprint.content.body;
+
+    for (const block of blocks) {
+        let blockContent = block.content;
+
+        // Replace {{VARIABLES}}
+        blockContent = expandVariables(blockContent, vars);
+
+        // Resolve spintax {A|B|C}
+        blockContent = resolver.resolve(blockContent);
+
+        fullHtml += blockContent + '\n\n';
+    }
+
+    // 2. Generate variation hash for uniqueness check
+    const variationHash = resolver.getChoicesHash();
+    const varsHash = crypto.createHash('sha256')
+        .update(JSON.stringify(vars))
+        .digest('hex')
+        .substring(0, 32);
+    const combinedHash = crypto.createHash('sha256')
+        .update(variationHash + varsHash)
+        .digest('hex')
+        .substring(0, 64);
+
+    // 3. Check if this variation already exists
+    const existingCheck = await pool.query(
+        `SELECT id FROM variation_registry WHERE variation_hash = $1`,
+        [combinedHash]
+    );
+
+    if (existingCheck.rows.length > 0) {
+        console.log(`⏭️  Skipping duplicate: ${vars.CITY}`);
+        return null;
+    }
+
+    // 4. Generate title and slug
+    const title = expandVariables(resolver.resolve(blueprint.asset_name), vars);
+    const slug = generateSlug(vars);
+
+    // 5. Generate meta description
+    const metaDesc = expandVariables(
+        resolver.resolve(blueprint.meta_description),
+        vars
+    );
+
+    // 6. Insert post
+    const postResult = await pool.query(
+        `INSERT INTO posts (
+      site_id, title, slug, content, excerpt, status, 
+      published_at, created_at
+    ) VALUES ($1, $2, $3, $4, $5, 'published', NOW(), NOW())
+    RETURNING id`,
+        [
+            siteId,
+            title,
+            slug,
+            fullHtml,
+            metaDesc.substring(0, 300)
+        ]
+    );
+
+    const postId = postResult.rows[0].id;
+
+    // 7. Record the variation
+    await pool.query(
+        `INSERT INTO variation_registry (
+      campaign_id, variation_hash, resolved_variables, spintax_choices, post_id
+    ) VALUES ($1, $2, $3, $4, $5)`,
+        [
+            campaignId,
+            combinedHash,
+            JSON.stringify(vars),
+            JSON.stringify(resolver.getChoices()),
+            postId
+        ]
+    );
+
+    // 8. Update block usage stats
+    await updateBlockUsageStats(blocks, resolver.getChoices());
+
+    return fullHtml;
+}
+
+/**
+ * Update usage statistics for blocks and spintax choices
+ */
+async function updateBlockUsageStats(blocks: any[], choices: any[]) {
+    // Track which blocks were used
+    for (const block of blocks) {
+        await pool.query(
+            `INSERT INTO block_usage_stats (content_fragment_id, block_type, total_uses, last_used_at)
+       SELECT id, $1, 1, NOW()
+       FROM content_fragments
+       WHERE block_type = $1
+       ON CONFLICT (content_fragment_id) DO UPDATE SET
+         total_uses = block_usage_stats.total_uses + 1,
+         last_used_at = NOW()`,
+            [block.block_type.replace(/ \(\d+\)$/, '')]
+        );
+    }
+
+    // Track spintax variation usage
+    for (const choice of choices) {
+        await pool.query(
+            `INSERT INTO spintax_variation_stats (
+        content_fragment_id, variation_path, variation_text, use_count, last_used_at
+      )
+      SELECT cf.id, $1, $2, 1, NOW()
+      FROM content_fragments cf
+      LIMIT 1
+      ON CONFLICT DO NOTHING`,
+            [choice.path, choice.chosen]
+        );
+    }
+}
+
+/**
+ * Generate URL-safe slug from variables
+ */
+function generateSlug(vars: Record<string, string>): string {
+    const city = (vars.CITY || 'city').toLowerCase().replace(/[^a-z0-9]+/g, '-');
+    const state = (vars.STATE || 'state').toLowerCase().replace(/[^a-z0-9]+/g, '-');
+    return `${city}-${state}-${Date.now()}`;
+}
+
+// Worker event handlers
+contentGeneratorWorker.on('completed', (job) => {
+    console.log(`✅ Job ${job.id} completed:`, job.returnvalue);
+});
+
+contentGeneratorWorker.on('failed', (job, err) => {
+    console.error(`❌ Job ${job?.id} failed:`, err);
+});
+
+contentGeneratorWorker.on('error', (err) => {
+    console.error('💥 Worker error:', err);
+});
+
+console.log('🚀 Content Generator Worker started');