-
Notifications
You must be signed in to change notification settings - Fork 4
Add inactive user re-engagement rollout script #29
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from all commits
0c7fdda
6b0da99
9c533c5
08482d4
3112ec6
647cffb
3c7f560
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,55 @@ | ||
| # CLI V1 Rollout - Inactive User Re-engagement | ||
|
|
||
| Grant $1 credit (7-day expiry) to users who used Kilo before but not in the last 30 days. | ||
|
|
||
| ## Two-Phase Approach | ||
|
|
||
| 1. **Phase 1 (cohort):** Tags eligible users into `cli-v1-rollout` cohort via SQL | ||
| 2. **Phase 2 (grant):** Grants credits to cohort members | ||
|
|
||
| Both scripts are idempotent - safe to re-run. | ||
|
|
||
| ## Running the Scripts | ||
|
|
||
| **Note:** Production env vars are required. Test `vercel env run` first: | ||
|
|
||
| ```bash | ||
| vercel env run --environment=production -- node -e "console.log(process.env.POSTGRES_URL ? 'OK' : 'NOT SET')" | ||
| ``` | ||
|
|
||
| ### Phase 1: Tag Cohort | ||
|
|
||
| ```bash | ||
| # Dry run (default) - shows count of users that would be tagged | ||
| vercel env run --environment=production -- pnpm script src/scripts/d2025-02-04_cli-v1-rollout-cohort.ts | ||
|
|
||
| # Apply | ||
| vercel env run --environment=production -- pnpm script src/scripts/d2025-02-04_cli-v1-rollout-cohort.ts --apply | ||
| ``` | ||
|
|
||
| ### Phase 2: Grant Credits | ||
|
|
||
| ```bash | ||
| # Dry run (default) - shows count of cohort members | ||
| vercel env run --environment=production -- pnpm script src/scripts/d2025-02-04_cli-v1-rollout-grant.ts | ||
|
|
||
| # Apply | ||
| vercel env run --environment=production -- pnpm script src/scripts/d2025-02-04_cli-v1-rollout-grant.ts --apply | ||
| ``` | ||
|
|
||
| ## Rollout Checklist | ||
|
|
||
| 1. [ ] Test `vercel env run` with simple command | ||
| 2. [ ] Run Phase 1 dry run, review user count | ||
| 3. [ ] Run Phase 1 with `--apply` | ||
| 4. [ ] Run Phase 2 dry run, verify cohort count matches | ||
| 5. [ ] Run Phase 2 with `--apply` | ||
| 6. [ ] Verify credits granted in admin panel | ||
|
|
||
| ## Fallback (if vercel env run doesn't work) | ||
|
|
||
| ```bash | ||
| vercel env pull .env.production | ||
| source .env.production # or use dotenv-cli | ||
| pnpm script src/scripts/d2025-02-04_cli-v1-rollout-cohort.ts | ||
| ``` |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,97 @@ | ||
| import { db } from '@/lib/drizzle'; | ||
| import { kilocode_users, microdollar_usage } from '@/db/schema'; | ||
| import { sql } from 'drizzle-orm'; | ||
|
|
||
| /** | ||
| * Phase 1: Tag eligible inactive users into the 'cli-v1-rollout' cohort. | ||
| * | ||
| * Eligible = non-blocked users who have used Kilo before but not in the last 30 days. | ||
| * | ||
| * Uses subqueries so the database handles the filtering — no large arrays | ||
| * materialized in application memory. | ||
| * | ||
| * Idempotent: users already in the cohort are skipped via the | ||
| * NOT (cohorts ? <cohort_name>) guard. | ||
| * | ||
| * Usage: | ||
| * pnpm script src/scripts/d2025-02-04_cli-v1-rollout-cohort.ts # dry run | ||
| * pnpm script src/scripts/d2025-02-04_cli-v1-rollout-cohort.ts --apply # apply | ||
| */ | ||
|
|
||
| const INACTIVE_DAYS = 30; | ||
| const COHORT_NAME = 'cli-v1-rollout'; | ||
|
|
||
| export async function tagInactiveUsersIntoCohort(options: { | ||
| cohortName: string; | ||
| inactiveDays: number; | ||
| dryRun: boolean; | ||
| }): Promise<{ tagged: number }> { | ||
| const cutoffIso = new Date(Date.now() - options.inactiveDays * 24 * 60 * 60 * 1000).toISOString(); | ||
| const now = Date.now(); | ||
|
|
||
| const everUsed = sql`(SELECT DISTINCT ${microdollar_usage.kilo_user_id} FROM ${microdollar_usage})`; | ||
| const recentlyActive = sql`(SELECT DISTINCT ${microdollar_usage.kilo_user_id} FROM ${microdollar_usage} WHERE ${microdollar_usage.created_at} >= ${cutoffIso})`; | ||
|
|
||
| if (options.dryRun) { | ||
| const countResult = await db.execute<{ count: string }>(sql` | ||
| SELECT COUNT(*) AS count | ||
| FROM ${kilocode_users} | ||
| WHERE ${kilocode_users.blocked_reason} IS NULL | ||
| AND ${kilocode_users.id} IN ${everUsed} | ||
| AND ${kilocode_users.id} NOT IN ${recentlyActive} | ||
| AND NOT (${kilocode_users.cohorts} ? ${options.cohortName}) | ||
| `); | ||
|
|
||
| return { tagged: Number(countResult.rows[0].count) }; | ||
| } | ||
|
|
||
| const result = await db.execute<{ count: string }>(sql` | ||
| WITH updated AS ( | ||
| UPDATE ${kilocode_users} | ||
| SET cohorts = cohorts || jsonb_build_object(${options.cohortName}::text, ${now}::bigint) | ||
| WHERE ${kilocode_users.blocked_reason} IS NULL | ||
| AND ${kilocode_users.id} IN ${everUsed} | ||
| AND ${kilocode_users.id} NOT IN ${recentlyActive} | ||
| AND NOT (cohorts ? ${options.cohortName}::text) | ||
| RETURNING ${kilocode_users.id} | ||
| ) | ||
| SELECT COUNT(*) AS count FROM updated | ||
| `); | ||
|
|
||
| return { tagged: Number(result.rows[0].count) }; | ||
| } | ||
|
|
||
| async function run() { | ||
| const isDryRun = !process.argv.includes('--apply'); | ||
|
|
||
| console.log('Phase 1: Tagging inactive users into cohort...\n'); | ||
| if (isDryRun) { | ||
| console.log('DRY RUN MODE - No changes will be made'); | ||
| console.log('Run with --apply flag to write cohort tags\n'); | ||
| } | ||
|
|
||
| const { tagged } = await tagInactiveUsersIntoCohort({ | ||
| cohortName: COHORT_NAME, | ||
| inactiveDays: INACTIVE_DAYS, | ||
| dryRun: isDryRun, | ||
| }); | ||
|
|
||
| if (isDryRun) { | ||
| console.log(`Would tag ${tagged} users into cohort '${COHORT_NAME}'`); | ||
| } else { | ||
| console.log(`Tagged ${tagged} users into cohort '${COHORT_NAME}'`); | ||
| } | ||
| } | ||
|
|
||
| // Only run if executed directly (not imported as a module for testing) | ||
| if (require.main === module || process.argv[1]?.endsWith('d2025-02-04_cli-v1-rollout-cohort.ts')) { | ||
| run() | ||
| .then(() => { | ||
| console.log('\nScript completed successfully'); | ||
| process.exit(0); | ||
| }) | ||
| .catch(error => { | ||
| console.error('\nScript failed:', error); | ||
| process.exit(1); | ||
| }); | ||
| } | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,135 @@ | ||
| import { db } from '@/lib/drizzle'; | ||
| import { kilocode_users, type User } from '@/db/schema'; | ||
| import { sql, gt, and, isNull } from 'drizzle-orm'; | ||
| import { grantCreditForCategory } from '@/lib/promotionalCredits'; | ||
| import * as fs from 'node:fs/promises'; | ||
| import * as path from 'node:path'; | ||
|
|
||
| /** | ||
| * Phase 2: Grant $1 credits to all users in the 'cli-v1-rollout' cohort. | ||
| * | ||
| * Reads users tagged by Phase 1 (cohort script) and grants them credits. | ||
| * Idempotent — safe to re-run; users who already received the credit are skipped. | ||
| * | ||
| * Usage: | ||
| * pnpm script src/scripts/d2025-02-04_cli-v1-rollout-grant.ts # dry run | ||
| * pnpm script src/scripts/d2025-02-04_cli-v1-rollout-grant.ts --apply # apply | ||
| */ | ||
|
|
||
| const COHORT_NAME = 'cli-v1-rollout'; | ||
|
|
||
| export type GrantCohortCreditsResult = { | ||
| granted: number; | ||
| skipped: number; | ||
| failed: number; | ||
| failedUserIds: string[]; | ||
| }; | ||
|
|
||
| export async function grantCreditsToCohort(options: { | ||
| cohortName: string; | ||
| creditCategory: string; | ||
| }): Promise<GrantCohortCreditsResult> { | ||
| let granted = 0; | ||
| let skipped = 0; | ||
| let failed = 0; | ||
| const failedUserIds: string[] = []; | ||
|
|
||
| let lastUserId: string | null = null; | ||
| let hasMore = true; | ||
|
|
||
| while (hasMore) { | ||
| const cohortFilter = sql`${kilocode_users.cohorts} ? ${options.cohortName}`; | ||
| const notBlocked = isNull(kilocode_users.blocked_reason); | ||
| const baseFilter = and(cohortFilter, notBlocked); | ||
| const users: User[] = await db.query.kilocode_users.findMany({ | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. WARNING: Blocked users may still receive credits
|
||
| where: lastUserId ? and(baseFilter, gt(kilocode_users.id, lastUserId)) : baseFilter, | ||
| orderBy: (kilocode_users, { asc }) => [asc(kilocode_users.id)], | ||
| limit: 1000, | ||
| }); | ||
|
|
||
| if (users.length === 0) { | ||
| hasMore = false; | ||
| break; | ||
| } | ||
|
|
||
| lastUserId = users[users.length - 1].id; | ||
| hasMore = users.length === 1000; | ||
|
|
||
| for (const user of users) { | ||
| try { | ||
| const result = await grantCreditForCategory(user, { | ||
| credit_category: options.creditCategory, | ||
| counts_as_selfservice: false, | ||
| }); | ||
|
|
||
| if (!result.success) { | ||
| if (result.message.includes('already been applied')) { | ||
| skipped++; | ||
| } else { | ||
| failed++; | ||
| failedUserIds.push(user.id); | ||
| } | ||
| } else { | ||
| granted++; | ||
| } | ||
| } catch { | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. SUGGESTION: Swallowed errors make grant failures hard to diagnose
|
||
| failed++; | ||
| failedUserIds.push(user.id); | ||
| } | ||
| } | ||
| } | ||
|
|
||
| return { granted, skipped, failed, failedUserIds }; | ||
| } | ||
|
|
||
| async function run() { | ||
| const isDryRun = !process.argv.includes('--apply'); | ||
|
|
||
| console.log('Phase 2: Granting credits to cohort members...'); | ||
| console.log(`Credit: $1 with 7-day expiry for users in cohort '${COHORT_NAME}'\n`); | ||
|
|
||
| if (isDryRun) { | ||
| console.log('DRY RUN MODE - No changes will be made'); | ||
| console.log('Run with --apply flag to grant credits\n'); | ||
|
|
||
| // In dry run, just count cohort members | ||
| const cohortFilter = sql`${kilocode_users.cohorts} ? ${COHORT_NAME}`; | ||
| const users = await db.query.kilocode_users.findMany({ where: cohortFilter }); | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. WARNING: Dry-run loads entire cohort into memory In dry-run mode this does |
||
| console.log(`Would grant credits to ${users.length} cohort members`); | ||
| return; | ||
| } | ||
|
|
||
| const scriptStartTime = Date.now(); | ||
| const result = await grantCreditsToCohort({ | ||
| cohortName: COHORT_NAME, | ||
| creditCategory: COHORT_NAME, | ||
| }); | ||
|
|
||
| const totalSeconds = (Date.now() - scriptStartTime) / 1000; | ||
| console.log('\n' + '='.repeat(60)); | ||
| console.log('Done!'); | ||
| console.log(` Granted: ${result.granted}`); | ||
| console.log(` Already applied: ${result.skipped}`); | ||
| console.log(` Failed: ${result.failed}`); | ||
| console.log(` Time: ${totalSeconds.toFixed(1)}s`); | ||
|
|
||
| if (result.failedUserIds.length > 0) { | ||
| const logFileName = `failed-users-cli-v1-rollout-${new Date().toISOString().replace(/[:.]/g, '-')}.log`; | ||
| const logFilePath = path.join(process.cwd(), logFileName); | ||
| await fs.writeFile(logFilePath, result.failedUserIds.join('\n') + '\n', 'utf-8'); | ||
| console.log(`\n${result.failedUserIds.length} failed user IDs written to: ${logFileName}`); | ||
| } | ||
| } | ||
|
|
||
| // Only run if executed directly (not imported as a module for testing) | ||
| if (require.main === module || process.argv[1]?.endsWith('d2025-02-04_cli-v1-rollout-grant.ts')) { | ||
| run() | ||
| .then(() => { | ||
| console.log('\nScript completed successfully'); | ||
| process.exit(0); | ||
| }) | ||
| .catch(error => { | ||
| console.error('\nScript failed:', error); | ||
| process.exit(1); | ||
| }); | ||
| } | ||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
WARNING: Cohort selection counts any
microdollar_usagerows (including $0 / failed requests)The SQL treats any
microdollar_usagerow as "used Kilo". If $0-cost rows include auth failures (e.g. OpenRouter 401s), users who never successfully used Kilo could be tagged, and recent failed requests could exclude otherwise-eligible users. Consider filtering to meaningful usage (e.g.WHERE cost > 0, and possiblyorganization_id IS NULLdepending on the intended cohort).