1231 lines
32 KiB
JavaScript
1231 lines
32 KiB
JavaScript
|
#!/usr/bin/env node
|
|||
|
|
|
|||
|
|
const { execSync } = require('child_process');
|
|||
|
|
const fs = require('fs');
|
|||
|
|
const path = require('path');
|
|||
|
|
|
|||
|
|
const COLORS = {
|
|||
|
|
reset: '\x1b[0m',
|
|||
|
|
bright: '\x1b[1m',
|
|||
|
|
red: '\x1b[31m',
|
|||
|
|
green: '\x1b[32m',
|
|||
|
|
yellow: '\x1b[33m',
|
|||
|
|
blue: '\x1b[34m',
|
|||
|
|
magenta: '\x1b[35m',
|
|||
|
|
cyan: '\x1b[36m',
|
|||
|
|
};
|
|||
|
|
|
|||
|
|
function log(message, color = COLORS.reset) {
|
|||
|
|
console.log(`${color}${message}${COLORS.reset}`);
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function logStep(step, message) {
|
|||
|
|
console.log(`\n${COLORS.cyan}[${step}]${COLORS.reset} ${message}`);
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function logSuccess(message) {
|
|||
|
|
console.log(`${COLORS.green}✅${COLORS.reset} ${message}`);
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function logError(message) {
|
|||
|
|
console.log(`${COLORS.red}❌${COLORS.reset} ${message}`);
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function logWarning(message) {
|
|||
|
|
console.log(`${COLORS.yellow}⚠️${COLORS.reset} ${message}`);
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function executeCommand(command, description) {
|
|||
|
|
try {
|
|||
|
|
logStep('EXEC', description);
|
|||
|
|
const result = execSync(command, {
|
|||
|
|
encoding: 'utf8',
|
|||
|
|
stdio: 'inherit'
|
|||
|
|
});
|
|||
|
|
logSuccess(description);
|
|||
|
|
return true;
|
|||
|
|
} catch (error) {
|
|||
|
|
logError(`${description} failed: ${error.message}`);
|
|||
|
|
return false;
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function generateDeploymentManifest() {
|
|||
|
|
logStep(1, 'Generating Deployment Manifest');
|
|||
|
|
|
|||
|
|
const manifest = {
|
|||
|
|
version: '1.0.0',
|
|||
|
|
deployment: {
|
|||
|
|
environment: process.env.NODE_ENV || 'production',
|
|||
|
|
region: 'us-east-1',
|
|||
|
|
timestamp: new Date().toISOString()
|
|||
|
|
},
|
|||
|
|
services: {
|
|||
|
|
server: {
|
|||
|
|
name: 'crawlful-hub-server',
|
|||
|
|
image: 'crawlful/hub-server:latest',
|
|||
|
|
replicas: 3,
|
|||
|
|
resources: {
|
|||
|
|
cpu: '2',
|
|||
|
|
memory: '4Gi'
|
|||
|
|
},
|
|||
|
|
ports: [
|
|||
|
|
{ containerPort: 3000, servicePort: 80, protocol: 'TCP' }
|
|||
|
|
],
|
|||
|
|
env: [
|
|||
|
|
{ name: 'NODE_ENV', value: 'production' },
|
|||
|
|
{ name: 'PORT', value: '3000' },
|
|||
|
|
{ name: 'DB_HOST', valueFrom: 'secret:db-host' },
|
|||
|
|
{ name: 'DB_PASSWORD', valueFrom: 'secret:db-password' },
|
|||
|
|
{ name: 'REDIS_HOST', valueFrom: 'secret:redis-host' },
|
|||
|
|
{ name: 'JWT_SECRET', valueFrom: 'secret:jwt-secret' }
|
|||
|
|
],
|
|||
|
|
healthCheck: {
|
|||
|
|
path: '/health',
|
|||
|
|
interval: 30,
|
|||
|
|
timeout: 5,
|
|||
|
|
failureThreshold: 3
|
|||
|
|
}
|
|||
|
|
},
|
|||
|
|
dashboard: {
|
|||
|
|
name: 'crawlful-hub-dashboard',
|
|||
|
|
image: 'crawlful/hub-dashboard:latest',
|
|||
|
|
replicas: 2,
|
|||
|
|
resources: {
|
|||
|
|
cpu: '1',
|
|||
|
|
memory: '2Gi'
|
|||
|
|
},
|
|||
|
|
ports: [
|
|||
|
|
{ containerPort: 80, servicePort: 80, protocol: 'TCP' }
|
|||
|
|
],
|
|||
|
|
healthCheck: {
|
|||
|
|
path: '/',
|
|||
|
|
interval: 30,
|
|||
|
|
timeout: 5,
|
|||
|
|
failureThreshold: 3
|
|||
|
|
}
|
|||
|
|
},
|
|||
|
|
websocket: {
|
|||
|
|
name: 'crawlful-hub-websocket',
|
|||
|
|
image: 'crawlful/hub-server:latest',
|
|||
|
|
replicas: 2,
|
|||
|
|
resources: {
|
|||
|
|
cpu: '0.5',
|
|||
|
|
memory: '1Gi'
|
|||
|
|
},
|
|||
|
|
ports: [
|
|||
|
|
{ containerPort: 8085, servicePort: 8085, protocol: 'TCP' }
|
|||
|
|
],
|
|||
|
|
env: [
|
|||
|
|
{ name: 'SERVICE_TYPE', value: 'websocket' }
|
|||
|
|
]
|
|||
|
|
}
|
|||
|
|
},
|
|||
|
|
databases: {
|
|||
|
|
mysql: {
|
|||
|
|
engine: 'mysql',
|
|||
|
|
version: '8.0',
|
|||
|
|
instanceClass: 'db.t3.large',
|
|||
|
|
allocatedStorage: 100,
|
|||
|
|
storageType: 'gp2',
|
|||
|
|
multiAZ: true,
|
|||
|
|
backupRetention: 7,
|
|||
|
|
parameters: {
|
|||
|
|
'max_connections': '500',
|
|||
|
|
'innodb_buffer_pool_size': '{DBInstanceClassMemory*3/4}'
|
|||
|
|
}
|
|||
|
|
},
|
|||
|
|
redis: {
|
|||
|
|
engine: 'redis',
|
|||
|
|
version: '6.x',
|
|||
|
|
nodeType: 'cache.m5.large',
|
|||
|
|
numCacheNodes: 3,
|
|||
|
|
replicationGroup: true,
|
|||
|
|
automaticFailover: true,
|
|||
|
|
engineLogDelivery: {
|
|||
|
|
destination: 'cloudwatch-logs',
|
|||
|
|
format: 'json'
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
},
|
|||
|
|
networking: {
|
|||
|
|
vpc: {
|
|||
|
|
cidr: '10.0.0.0/16',
|
|||
|
|
subnets: {
|
|||
|
|
public: ['10.0.1.0/24', '10.0.2.0/24'],
|
|||
|
|
private: ['10.0.10.0/24', '10.0.11.0/24']
|
|||
|
|
}
|
|||
|
|
},
|
|||
|
|
loadBalancer: {
|
|||
|
|
type: 'application',
|
|||
|
|
scheme: 'internet-facing',
|
|||
|
|
sslCertificate: 'arn:aws:acm:us-east-1:123456789012:certificate/abc123'
|
|||
|
|
},
|
|||
|
|
securityGroups: [
|
|||
|
|
{
|
|||
|
|
name: 'web-sg',
|
|||
|
|
description: 'Web server security group',
|
|||
|
|
ingress: [
|
|||
|
|
{ protocol: 'TCP', port: 80, source: '0.0.0.0/0' },
|
|||
|
|
{ protocol: 'TCP', port: 443, source: '0.0.0.0/0' }
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
name: 'db-sg',
|
|||
|
|
description: 'Database security group',
|
|||
|
|
ingress: [
|
|||
|
|
{ protocol: 'TCP', port: 3306, source: 'web-sg' },
|
|||
|
|
{ protocol: 'TCP', port: 6379, source: 'web-sg' }
|
|||
|
|
]
|
|||
|
|
}
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
monitoring: {
|
|||
|
|
cloudWatch: {
|
|||
|
|
enabled: true,
|
|||
|
|
metrics: [
|
|||
|
|
{ name: 'CPUUtilization', namespace: 'AWS/ECS' },
|
|||
|
|
{ name: 'MemoryUtilization', namespace: 'AWS/ECS' },
|
|||
|
|
{ name: 'RequestCount', namespace: 'AWS/ApplicationELB' },
|
|||
|
|
{ name: 'TargetResponseTime', namespace: 'AWS/ApplicationELB' }
|
|||
|
|
],
|
|||
|
|
alarms: [
|
|||
|
|
{
|
|||
|
|
name: 'HighCPUUtilization',
|
|||
|
|
metric: 'CPUUtilization',
|
|||
|
|
threshold: 80,
|
|||
|
|
comparison: 'GreaterThanThreshold',
|
|||
|
|
period: 300,
|
|||
|
|
evaluationPeriods: 2
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
name: 'HighMemoryUtilization',
|
|||
|
|
metric: 'MemoryUtilization',
|
|||
|
|
threshold: 85,
|
|||
|
|
comparison: 'GreaterThanThreshold',
|
|||
|
|
period: 300,
|
|||
|
|
evaluationPeriods: 2
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
name: 'HighErrorRate',
|
|||
|
|
metric: 'HTTPCode_Target_5XX_Count',
|
|||
|
|
threshold: 10,
|
|||
|
|
comparison: 'GreaterThanThreshold',
|
|||
|
|
period: 60,
|
|||
|
|
evaluationPeriods: 1
|
|||
|
|
}
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
xray: {
|
|||
|
|
enabled: true,
|
|||
|
|
samplingRate: 0.1
|
|||
|
|
}
|
|||
|
|
},
|
|||
|
|
secrets: {
|
|||
|
|
db: {
|
|||
|
|
host: 'mysql.crawlful.internal',
|
|||
|
|
port: 3306,
|
|||
|
|
user: 'crawlful_admin',
|
|||
|
|
database: 'crawlful_hub'
|
|||
|
|
},
|
|||
|
|
redis: {
|
|||
|
|
host: 'redis.crawlful.internal',
|
|||
|
|
port: 6379
|
|||
|
|
},
|
|||
|
|
jwt: {
|
|||
|
|
secret: 'CHANGE_ME_IN_PRODUCTION'
|
|||
|
|
},
|
|||
|
|
api: {
|
|||
|
|
keys: {
|
|||
|
|
amazon: 'CHANGE_ME',
|
|||
|
|
ebay: 'CHANGE_ME',
|
|||
|
|
shopify: 'CHANGE_ME',
|
|||
|
|
shopee: 'CHANGE_ME',
|
|||
|
|
tiktok: 'CHANGE_ME'
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
};
|
|||
|
|
|
|||
|
|
const manifestPath = path.join(process.cwd(), 'deployment-manifest.json');
|
|||
|
|
fs.writeFileSync(manifestPath, JSON.stringify(manifest, null, 2));
|
|||
|
|
logSuccess(`Deployment manifest generated: ${manifestPath}`);
|
|||
|
|
return true;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function generateDockerCompose() {
|
|||
|
|
logStep(2, 'Generating Docker Compose Configuration');
|
|||
|
|
|
|||
|
|
const dockerCompose = {
|
|||
|
|
version: '3.8',
|
|||
|
|
services: {
|
|||
|
|
server: {
|
|||
|
|
build: {
|
|||
|
|
context: './server',
|
|||
|
|
dockerfile: 'Dockerfile'
|
|||
|
|
},
|
|||
|
|
container_name: 'crawlful-hub-server',
|
|||
|
|
ports: ['3000:3000'],
|
|||
|
|
environment: [
|
|||
|
|
'NODE_ENV=production',
|
|||
|
|
'PORT=3000',
|
|||
|
|
'DB_HOST=mysql',
|
|||
|
|
'DB_PORT=3306',
|
|||
|
|
'DB_USER=crawlful',
|
|||
|
|
'DB_PASSWORD=crawlful_password',
|
|||
|
|
'DB_NAME=crawlful_hub',
|
|||
|
|
'REDIS_HOST=redis',
|
|||
|
|
'REDIS_PORT=6379',
|
|||
|
|
'JWT_SECRET=your-secret-key-change-in-production'
|
|||
|
|
],
|
|||
|
|
depends_on: ['mysql', 'redis'],
|
|||
|
|
volumes: ['./server:/app', '/app/node_modules'],
|
|||
|
|
restart: 'unless-stopped',
|
|||
|
|
healthcheck: {
|
|||
|
|
test: ['CMD', 'curl', '-f', 'http://localhost:3000/health'],
|
|||
|
|
interval: '30s',
|
|||
|
|
timeout: '10s',
|
|||
|
|
retries: 3
|
|||
|
|
}
|
|||
|
|
},
|
|||
|
|
dashboard: {
|
|||
|
|
build: {
|
|||
|
|
context: './dashboard',
|
|||
|
|
dockerfile: 'Dockerfile'
|
|||
|
|
},
|
|||
|
|
container_name: 'crawlful-hub-dashboard',
|
|||
|
|
ports: ['80:80'],
|
|||
|
|
depends_on: ['server'],
|
|||
|
|
volumes: ['./dashboard:/app', '/app/node_modules'],
|
|||
|
|
restart: 'unless-stopped',
|
|||
|
|
healthcheck: {
|
|||
|
|
test: ['CMD', 'curl', '-f', 'http://localhost/'],
|
|||
|
|
interval: '30s',
|
|||
|
|
timeout: '10s',
|
|||
|
|
retries: 3
|
|||
|
|
}
|
|||
|
|
},
|
|||
|
|
websocket: {
|
|||
|
|
build: {
|
|||
|
|
context: './server',
|
|||
|
|
dockerfile: 'Dockerfile'
|
|||
|
|
},
|
|||
|
|
container_name: 'crawlful-hub-websocket',
|
|||
|
|
command: 'npm run websocket',
|
|||
|
|
ports: ['8085:8085'],
|
|||
|
|
environment: [
|
|||
|
|
'NODE_ENV=production',
|
|||
|
|
'WS_PORT=8085',
|
|||
|
|
'REDIS_HOST=redis',
|
|||
|
|
'REDIS_PORT=6379'
|
|||
|
|
],
|
|||
|
|
depends_on: ['redis'],
|
|||
|
|
restart: 'unless-stopped'
|
|||
|
|
},
|
|||
|
|
mysql: {
|
|||
|
|
image: 'mysql:8.0',
|
|||
|
|
container_name: 'crawlful-mysql',
|
|||
|
|
ports: ['3306:3306'],
|
|||
|
|
environment: [
|
|||
|
|
'MYSQL_ROOT_PASSWORD=root_password',
|
|||
|
|
'MYSQL_DATABASE=crawlful_hub',
|
|||
|
|
'MYSQL_USER=crawlful',
|
|||
|
|
'MYSQL_PASSWORD=crawlful_password'
|
|||
|
|
],
|
|||
|
|
volumes: [
|
|||
|
|
'./scripts/db-init.sql:/docker-entrypoint-initdb.d/init.sql',
|
|||
|
|
'mysql_data:/var/lib/mysql'
|
|||
|
|
],
|
|||
|
|
restart: 'unless-stopped',
|
|||
|
|
healthcheck: {
|
|||
|
|
test: ['CMD', 'mysqladmin', 'ping', '-h', 'localhost'],
|
|||
|
|
interval: '10s',
|
|||
|
|
timeout: '5s',
|
|||
|
|
retries: 5
|
|||
|
|
}
|
|||
|
|
},
|
|||
|
|
redis: {
|
|||
|
|
image: 'redis:7-alpine',
|
|||
|
|
container_name: 'crawlful-redis',
|
|||
|
|
ports: ['6379:6379'],
|
|||
|
|
volumes: ['redis_data:/data'],
|
|||
|
|
restart: 'unless-stopped',
|
|||
|
|
healthcheck: {
|
|||
|
|
test: ['CMD', 'redis-cli', 'ping'],
|
|||
|
|
interval: '10s',
|
|||
|
|
timeout: '5s',
|
|||
|
|
retries: 5
|
|||
|
|
}
|
|||
|
|
},
|
|||
|
|
prometheus: {
|
|||
|
|
image: 'prom/prometheus:latest',
|
|||
|
|
container_name: 'crawlful-prometheus',
|
|||
|
|
ports: ['9090:9090'],
|
|||
|
|
volumes: [
|
|||
|
|
'./monitoring/prometheus.yml:/etc/prometheus/prometheus.yml',
|
|||
|
|
'prometheus_data:/prometheus'
|
|||
|
|
],
|
|||
|
|
restart: 'unless-stopped'
|
|||
|
|
},
|
|||
|
|
grafana: {
|
|||
|
|
image: 'grafana/grafana:latest',
|
|||
|
|
container_name: 'crawlful-grafana',
|
|||
|
|
ports: ['3001:3000'],
|
|||
|
|
environment: [
|
|||
|
|
'GF_SECURITY_ADMIN_PASSWORD=admin'
|
|||
|
|
],
|
|||
|
|
volumes: [
|
|||
|
|
'grafana_data:/var/lib/grafana',
|
|||
|
|
'./monitoring/grafana/dashboards:/etc/grafana/provisioning/dashboards',
|
|||
|
|
'./monitoring/grafana/datasources:/etc/grafana/provisioning/datasources'
|
|||
|
|
],
|
|||
|
|
restart: 'unless-stopped'
|
|||
|
|
}
|
|||
|
|
},
|
|||
|
|
volumes: {
|
|||
|
|
mysql_data: {},
|
|||
|
|
redis_data: {},
|
|||
|
|
prometheus_data: {},
|
|||
|
|
grafana_data: {}
|
|||
|
|
},
|
|||
|
|
networks: {
|
|||
|
|
crawlful_network: {
|
|||
|
|
driver: 'bridge'
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
};
|
|||
|
|
|
|||
|
|
const dockerComposePath = path.join(process.cwd(), 'docker-compose.yml');
|
|||
|
|
fs.writeFileSync(dockerComposePath, JSON.stringify(dockerCompose, null, 2).replace(/"([^"]+)":/g, '$1:'));
|
|||
|
|
logSuccess(`Docker Compose configuration generated: ${dockerComposePath}`);
|
|||
|
|
return true;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function generateKubernetesManifests() {
|
|||
|
|
logStep(3, 'Generating Kubernetes Manifests');
|
|||
|
|
|
|||
|
|
const k8sDir = path.join(process.cwd(), 'k8s');
|
|||
|
|
if (!fs.existsSync(k8sDir)) {
|
|||
|
|
fs.mkdirSync(k8sDir, { recursive: true });
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
const namespace = {
|
|||
|
|
apiVersion: 'v1',
|
|||
|
|
kind: 'Namespace',
|
|||
|
|
metadata: {
|
|||
|
|
name: 'crawlful-hub'
|
|||
|
|
}
|
|||
|
|
};
|
|||
|
|
|
|||
|
|
const deployment = {
|
|||
|
|
apiVersion: 'apps/v1',
|
|||
|
|
kind: 'Deployment',
|
|||
|
|
metadata: {
|
|||
|
|
name: 'crawlful-hub-server',
|
|||
|
|
namespace: 'crawlful-hub'
|
|||
|
|
},
|
|||
|
|
spec: {
|
|||
|
|
replicas: 3,
|
|||
|
|
selector: {
|
|||
|
|
matchLabels: {
|
|||
|
|
app: 'crawlful-hub-server'
|
|||
|
|
}
|
|||
|
|
},
|
|||
|
|
template: {
|
|||
|
|
metadata: {
|
|||
|
|
labels: {
|
|||
|
|
app: 'crawlful-hub-server'
|
|||
|
|
}
|
|||
|
|
},
|
|||
|
|
spec: {
|
|||
|
|
containers: [
|
|||
|
|
{
|
|||
|
|
name: 'server',
|
|||
|
|
image: 'crawlful/hub-server:latest',
|
|||
|
|
ports: [
|
|||
|
|
{ containerPort: 3000 }
|
|||
|
|
],
|
|||
|
|
env: [
|
|||
|
|
{ name: 'NODE_ENV', value: 'production' },
|
|||
|
|
{ name: 'PORT', value: '3000' },
|
|||
|
|
{ name: 'DB_HOST', valueFrom: { secretKeyRef: { name: 'db-secret', key: 'host' } } },
|
|||
|
|
{ name: 'DB_PASSWORD', valueFrom: { secretKeyRef: { name: 'db-secret', key: 'password' } } },
|
|||
|
|
{ name: 'REDIS_HOST', valueFrom: { secretKeyRef: { name: 'redis-secret', key: 'host' } } },
|
|||
|
|
{ name: 'JWT_SECRET', valueFrom: { secretKeyRef: { name: 'jwt-secret', key: 'secret' } } }
|
|||
|
|
],
|
|||
|
|
resources: {
|
|||
|
|
requests: {
|
|||
|
|
cpu: '500m',
|
|||
|
|
memory: '1Gi'
|
|||
|
|
},
|
|||
|
|
limits: {
|
|||
|
|
cpu: '2',
|
|||
|
|
memory: '4Gi'
|
|||
|
|
}
|
|||
|
|
},
|
|||
|
|
livenessProbe: {
|
|||
|
|
httpGet: { path: '/health', port: 3000 },
|
|||
|
|
initialDelaySeconds: 30,
|
|||
|
|
periodSeconds: 10
|
|||
|
|
},
|
|||
|
|
readinessProbe: {
|
|||
|
|
httpGet: { path: '/health', port: 3000 },
|
|||
|
|
initialDelaySeconds: 5,
|
|||
|
|
periodSeconds: 5
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
]
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
};
|
|||
|
|
|
|||
|
|
const service = {
|
|||
|
|
apiVersion: 'v1',
|
|||
|
|
kind: 'Service',
|
|||
|
|
metadata: {
|
|||
|
|
name: 'crawlful-hub-server',
|
|||
|
|
namespace: 'crawlful-hub'
|
|||
|
|
},
|
|||
|
|
spec: {
|
|||
|
|
selector: {
|
|||
|
|
app: 'crawlful-hub-server'
|
|||
|
|
},
|
|||
|
|
ports: [
|
|||
|
|
{ port: 80, targetPort: 3000, protocol: 'TCP' }
|
|||
|
|
],
|
|||
|
|
type: 'LoadBalancer'
|
|||
|
|
}
|
|||
|
|
};
|
|||
|
|
|
|||
|
|
const ingress = {
|
|||
|
|
apiVersion: 'networking.k8s.io/v1',
|
|||
|
|
kind: 'Ingress',
|
|||
|
|
metadata: {
|
|||
|
|
name: 'crawlful-hub-ingress',
|
|||
|
|
namespace: 'crawlful-hub',
|
|||
|
|
annotations: {
|
|||
|
|
'kubernetes.io/ingress.class': 'nginx',
|
|||
|
|
'cert-manager.io/cluster-issuer': 'letsencrypt-prod'
|
|||
|
|
}
|
|||
|
|
},
|
|||
|
|
spec: {
|
|||
|
|
tls: [
|
|||
|
|
{
|
|||
|
|
hosts: ['api.crawlful.com'],
|
|||
|
|
secretName: 'crawlful-tls'
|
|||
|
|
}
|
|||
|
|
],
|
|||
|
|
rules: [
|
|||
|
|
{
|
|||
|
|
host: 'api.crawlful.com',
|
|||
|
|
http: {
|
|||
|
|
paths: [
|
|||
|
|
{
|
|||
|
|
path: '/',
|
|||
|
|
pathType: 'Prefix',
|
|||
|
|
backend: {
|
|||
|
|
service: {
|
|||
|
|
name: 'crawlful-hub-server',
|
|||
|
|
port: { number: 80 }
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
]
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
]
|
|||
|
|
}
|
|||
|
|
};
|
|||
|
|
|
|||
|
|
fs.writeFileSync(path.join(k8sDir, 'namespace.yaml'), JSON.stringify(namespace, null, 2));
|
|||
|
|
fs.writeFileSync(path.join(k8sDir, 'deployment.yaml'), JSON.stringify(deployment, null, 2));
|
|||
|
|
fs.writeFileSync(path.join(k8sDir, 'service.yaml'), JSON.stringify(service, null, 2));
|
|||
|
|
fs.writeFileSync(path.join(k8sDir, 'ingress.yaml'), JSON.stringify(ingress, null, 2));
|
|||
|
|
|
|||
|
|
logSuccess('Kubernetes manifests generated in k8s/ directory');
|
|||
|
|
return true;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function generateDeploymentScripts() {
|
|||
|
|
logStep(4, 'Generating Deployment Scripts');
|
|||
|
|
|
|||
|
|
const deployScript = `#!/bin/bash
|
|||
|
|
|
|||
|
|
# Crawlful Hub Deployment Script
|
|||
|
|
# This script handles the deployment of the Crawlful Hub system
|
|||
|
|
|
|||
|
|
set -e
|
|||
|
|
|
|||
|
|
echo "🚀 Starting Crawlful Hub Deployment..."
|
|||
|
|
|
|||
|
|
# Configuration
|
|||
|
|
ENVIRONMENT=\${1:-production}
|
|||
|
|
REGION=\${2:-us-east-1}
|
|||
|
|
PROJECT_NAME="crawlful-hub"
|
|||
|
|
|
|||
|
|
# Colors
|
|||
|
|
RED='\\033[0;31m'
|
|||
|
|
GREEN='\\033[0;32m'
|
|||
|
|
YELLOW='\\033[1;33m'
|
|||
|
|
NC='\\033[0m' # No Color
|
|||
|
|
|
|||
|
|
log_info() {
|
|||
|
|
echo -e "\${GREEN}ℹ\${NC} \$1"
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
log_warn() {
|
|||
|
|
echo -e "\${YELLOW}⚠\${NC} \$1"
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
log_error() {
|
|||
|
|
echo -e "\${RED}❌\${NC} \$1"
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
# Pre-deployment checks
|
|||
|
|
check_prerequisites() {
|
|||
|
|
log_info "Checking prerequisites..."
|
|||
|
|
|
|||
|
|
if ! command -v docker &> /dev/null; then
|
|||
|
|
log_error "Docker is not installed"
|
|||
|
|
exit 1
|
|||
|
|
fi
|
|||
|
|
|
|||
|
|
if ! command -v kubectl &> /dev/null; then
|
|||
|
|
log_error "kubectl is not installed"
|
|||
|
|
exit 1
|
|||
|
|
fi
|
|||
|
|
|
|||
|
|
log_info "Prerequisites check passed"
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
# Build Docker images
|
|||
|
|
build_images() {
|
|||
|
|
log_info "Building Docker images..."
|
|||
|
|
|
|||
|
|
docker build -t \${PROJECT_NAME}-server:latest ./server
|
|||
|
|
docker build -t \${PROJECT_NAME}-dashboard:latest ./dashboard
|
|||
|
|
|
|||
|
|
log_info "Docker images built successfully"
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
# Push images to registry
|
|||
|
|
push_images() {
|
|||
|
|
log_info "Pushing images to registry..."
|
|||
|
|
|
|||
|
|
# Add your registry push commands here
|
|||
|
|
# docker tag \${PROJECT_NAME}-server:latest your-registry/\${PROJECT_NAME}-server:latest
|
|||
|
|
# docker push your-registry/\${PROJECT_NAME}-server:latest
|
|||
|
|
|
|||
|
|
log_info "Images pushed successfully"
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
# Deploy to Kubernetes
|
|||
|
|
deploy_kubernetes() {
|
|||
|
|
log_info "Deploying to Kubernetes..."
|
|||
|
|
|
|||
|
|
kubectl apply -f k8s/namespace.yaml
|
|||
|
|
kubectl apply -f k8s/deployment.yaml
|
|||
|
|
kubectl apply -f k8s/service.yaml
|
|||
|
|
kubectl apply -f k8s/ingress.yaml
|
|||
|
|
|
|||
|
|
log_info "Kubernetes deployment completed"
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
# Run database migrations
|
|||
|
|
run_migrations() {
|
|||
|
|
log_info "Running database migrations..."
|
|||
|
|
|
|||
|
|
kubectl exec -it \$(kubectl get pods -n crawlful-hub -l app=crawlful-hub-server -o jsonpath='{.items[0].metadata.name}') -n crawlful-hub -- npm run migrate
|
|||
|
|
|
|||
|
|
log_info "Database migrations completed"
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
# Verify deployment
|
|||
|
|
verify_deployment() {
|
|||
|
|
log_info "Verifying deployment..."
|
|||
|
|
|
|||
|
|
kubectl rollout status deployment/crawlful-hub-server -n crawlful-hub
|
|||
|
|
|
|||
|
|
log_info "Deployment verification completed"
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
# Main deployment flow
|
|||
|
|
main() {
|
|||
|
|
check_prerequisites
|
|||
|
|
build_images
|
|||
|
|
push_images
|
|||
|
|
deploy_kubernetes
|
|||
|
|
run_migrations
|
|||
|
|
verify_deployment
|
|||
|
|
|
|||
|
|
echo -e "\${GREEN}✅ Deployment completed successfully!\${NC}"
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
main
|
|||
|
|
`;
|
|||
|
|
|
|||
|
|
const rollbackScript = `#!/bin/bash
|
|||
|
|
|
|||
|
|
# Crawlful Hub Rollback Script
|
|||
|
|
# This script handles rollback of the Crawlful Hub system
|
|||
|
|
|
|||
|
|
set -e
|
|||
|
|
|
|||
|
|
echo "🔄 Starting Crawlful Hub Rollback..."
|
|||
|
|
|
|||
|
|
# Configuration
|
|||
|
|
ENVIRONMENT=\${1:-production}
|
|||
|
|
PROJECT_NAME="crawlful-hub"
|
|||
|
|
|
|||
|
|
# Colors
|
|||
|
|
RED='\\033[0;31m'
|
|||
|
|
GREEN='\\033[0;32m'
|
|||
|
|
YELLOW='\\033[1;33m'
|
|||
|
|
NC='\\033[0m' # No Color
|
|||
|
|
|
|||
|
|
log_info() {
|
|||
|
|
echo -e "\${GREEN}ℹ\${NC} \$1"
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
log_warn() {
|
|||
|
|
echo -e "\${YELLOW}⚠\${NC} \$1"
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
log_error() {
|
|||
|
|
echo -e "\${RED}❌\${NC} \$1"
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
# Rollback Kubernetes deployment
|
|||
|
|
rollback_kubernetes() {
|
|||
|
|
log_info "Rolling back Kubernetes deployment..."
|
|||
|
|
|
|||
|
|
kubectl rollout undo deployment/crawlful-hub-server -n crawlful-hub
|
|||
|
|
|
|||
|
|
log_info "Kubernetes rollback completed"
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
# Verify rollback
|
|||
|
|
verify_rollback() {
|
|||
|
|
log_info "Verifying rollback..."
|
|||
|
|
|
|||
|
|
kubectl rollout status deployment/crawlful-hub-server -n crawlful-hub
|
|||
|
|
|
|||
|
|
log_info "Rollback verification completed"
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
# Main rollback flow
|
|||
|
|
main() {
|
|||
|
|
rollback_kubernetes
|
|||
|
|
verify_rollback
|
|||
|
|
|
|||
|
|
echo -e "\${GREEN}✅ Rollback completed successfully!\${NC}"
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
main
|
|||
|
|
`;
|
|||
|
|
|
|||
|
|
const scriptsDir = path.join(process.cwd(), 'scripts');
|
|||
|
|
if (!fs.existsSync(scriptsDir)) {
|
|||
|
|
fs.mkdirSync(scriptsDir, { recursive: true });
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
fs.writeFileSync(path.join(scriptsDir, 'deploy.sh'), deployScript);
|
|||
|
|
fs.writeFileSync(path.join(scriptsDir, 'rollback.sh'), rollbackScript);
|
|||
|
|
|
|||
|
|
logSuccess('Deployment scripts generated in scripts/ directory');
|
|||
|
|
return true;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function generateMonitoringConfig() {
|
|||
|
|
logStep(5, 'Generating Monitoring Configuration');
|
|||
|
|
|
|||
|
|
const monitoringDir = path.join(process.cwd(), 'monitoring');
|
|||
|
|
if (!fs.existsSync(monitoringDir)) {
|
|||
|
|
fs.mkdirSync(monitoringDir, { recursive: true });
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
const prometheusConfig = {
|
|||
|
|
global: {
|
|||
|
|
scrape_interval: '15s',
|
|||
|
|
evaluation_interval: '15s'
|
|||
|
|
},
|
|||
|
|
scrape_configs: [
|
|||
|
|
{
|
|||
|
|
job_name: 'crawlful-hub-server',
|
|||
|
|
static_configs: [
|
|||
|
|
{
|
|||
|
|
targets: ['server:3000'],
|
|||
|
|
labels: {
|
|||
|
|
service: 'crawlful-hub',
|
|||
|
|
environment: 'production'
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
job_name: 'crawlful-hub-dashboard',
|
|||
|
|
static_configs: [
|
|||
|
|
{
|
|||
|
|
targets: ['dashboard:80'],
|
|||
|
|
labels: {
|
|||
|
|
service: 'crawlful-hub-dashboard',
|
|||
|
|
environment: 'production'
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
job_name: 'mysql-exporter',
|
|||
|
|
static_configs: [
|
|||
|
|
{
|
|||
|
|
targets: ['mysql-exporter:9104'],
|
|||
|
|
labels: {
|
|||
|
|
service: 'mysql',
|
|||
|
|
environment: 'production'
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
job_name: 'redis-exporter',
|
|||
|
|
static_configs: [
|
|||
|
|
{
|
|||
|
|
targets: ['redis-exporter:9121'],
|
|||
|
|
labels: {
|
|||
|
|
service: 'redis',
|
|||
|
|
environment: 'production'
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
]
|
|||
|
|
}
|
|||
|
|
]
|
|||
|
|
};
|
|||
|
|
|
|||
|
|
const prometheusDir = path.join(monitoringDir, 'prometheus');
|
|||
|
|
if (!fs.existsSync(prometheusDir)) {
|
|||
|
|
fs.mkdirSync(prometheusDir, { recursive: true });
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
fs.writeFileSync(path.join(prometheusDir, 'prometheus.yml'), JSON.stringify(prometheusConfig, null, 2).replace(/"([^"]+)":/g, '$1:'));
|
|||
|
|
|
|||
|
|
logSuccess('Monitoring configuration generated in monitoring/ directory');
|
|||
|
|
return true;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function generateDocumentation() {
|
|||
|
|
logStep(6, 'Generating Deployment Documentation');
|
|||
|
|
|
|||
|
|
const docsDir = path.join(process.cwd(), 'docs', 'Deployment');
|
|||
|
|
if (!fs.existsSync(docsDir)) {
|
|||
|
|
fs.mkdirSync(docsDir, { recursive: true });
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
const deploymentGuide = `# Crawlful Hub Deployment Guide
|
|||
|
|
|
|||
|
|
## Overview
|
|||
|
|
This guide provides comprehensive instructions for deploying the Crawlful Hub system to production environments.
|
|||
|
|
|
|||
|
|
## Prerequisites
|
|||
|
|
|
|||
|
|
### Software Requirements
|
|||
|
|
- Docker 20.10+
|
|||
|
|
- Kubernetes 1.20+
|
|||
|
|
- kubectl 1.20+
|
|||
|
|
- Helm 3.0+
|
|||
|
|
- AWS CLI 2.0+ (for AWS deployment)
|
|||
|
|
|
|||
|
|
### Infrastructure Requirements
|
|||
|
|
- At least 3 nodes with 4 vCPU and 8GB RAM each
|
|||
|
|
- 100GB storage for MySQL
|
|||
|
|
- 10GB storage for Redis
|
|||
|
|
- Load balancer with SSL certificate
|
|||
|
|
|
|||
|
|
## Deployment Options
|
|||
|
|
|
|||
|
|
### Option 1: Docker Compose (Development/Testing)
|
|||
|
|
\`\`\`bash
|
|||
|
|
# Start all services
|
|||
|
|
docker-compose up -d
|
|||
|
|
|
|||
|
|
# View logs
|
|||
|
|
docker-compose logs -f
|
|||
|
|
|
|||
|
|
# Stop all services
|
|||
|
|
docker-compose down
|
|||
|
|
\`\`\`
|
|||
|
|
|
|||
|
|
### Option 2: Kubernetes (Production)
|
|||
|
|
\`\`\`bash
|
|||
|
|
# Deploy to Kubernetes
|
|||
|
|
./scripts/deploy.sh production us-east-1
|
|||
|
|
|
|||
|
|
# Check deployment status
|
|||
|
|
kubectl get pods -n crawlful-hub
|
|||
|
|
|
|||
|
|
# View logs
|
|||
|
|
kubectl logs -f deployment/crawlful-hub-server -n crawlful-hub
|
|||
|
|
\`\`\`
|
|||
|
|
|
|||
|
|
### Option 3: AWS EKS (Production)
|
|||
|
|
\`\`\`bash
|
|||
|
|
# Create EKS cluster
|
|||
|
|
eksctl create cluster --name crawlful-hub --region us-east-1
|
|||
|
|
|
|||
|
|
# Deploy to EKS
|
|||
|
|
kubectl apply -f k8s/
|
|||
|
|
|
|||
|
|
# Configure Load Balancer
|
|||
|
|
kubectl apply -f k8s/ingress.yaml
|
|||
|
|
\`\`\`
|
|||
|
|
|
|||
|
|
## Configuration
|
|||
|
|
|
|||
|
|
### Environment Variables
|
|||
|
|
Create a \`.env\` file with the following variables:
|
|||
|
|
|
|||
|
|
\`\`\`bash
|
|||
|
|
# Database
|
|||
|
|
DB_HOST=mysql.crawlful.internal
|
|||
|
|
DB_PORT=3306
|
|||
|
|
DB_USER=crawlful
|
|||
|
|
DB_PASSWORD=your_password
|
|||
|
|
DB_NAME=crawlful_hub
|
|||
|
|
|
|||
|
|
# Redis
|
|||
|
|
REDIS_HOST=redis.crawlful.internal
|
|||
|
|
REDIS_PORT=6379
|
|||
|
|
|
|||
|
|
# JWT
|
|||
|
|
JWT_SECRET=your_jwt_secret
|
|||
|
|
|
|||
|
|
# Application
|
|||
|
|
NODE_ENV=production
|
|||
|
|
PORT=3000
|
|||
|
|
WS_PORT=8085
|
|||
|
|
\`\`\`
|
|||
|
|
|
|||
|
|
### Secrets Management
|
|||
|
|
For production, use Kubernetes secrets:
|
|||
|
|
|
|||
|
|
\`\`\`bash
|
|||
|
|
# Create database secret
|
|||
|
|
kubectl create secret generic db-secret \\
|
|||
|
|
--from-literal=host=mysql.crawlful.internal \\
|
|||
|
|
--from-literal=password=your_password \\
|
|||
|
|
-n crawlful-hub
|
|||
|
|
|
|||
|
|
# Create JWT secret
|
|||
|
|
kubectl create secret generic jwt-secret \\
|
|||
|
|
--from-literal=secret=your_jwt_secret \\
|
|||
|
|
-n crawlful-hub
|
|||
|
|
\`\`\`
|
|||
|
|
|
|||
|
|
## Monitoring and Logging
|
|||
|
|
|
|||
|
|
### Prometheus Metrics
|
|||
|
|
Access Prometheus at \`http://your-domain:9090\`
|
|||
|
|
|
|||
|
|
### Grafana Dashboards
|
|||
|
|
Access Grafana at \`http://your-domain:3001\`
|
|||
|
|
- Default credentials: admin/admin
|
|||
|
|
|
|||
|
|
### CloudWatch Logs
|
|||
|
|
For AWS deployment, logs are automatically sent to CloudWatch.
|
|||
|
|
|
|||
|
|
## Backup and Recovery
|
|||
|
|
|
|||
|
|
### Database Backup
|
|||
|
|
\`\`\`bash
|
|||
|
|
# Create backup
|
|||
|
|
kubectl exec -it \$(kubectl get pods -n crawlful-hub -l app=mysql -o jsonpath='{.items[0].metadata.name}') -n crawlful-hub -- \\
|
|||
|
|
mysqldump -u root -p crawlful_hub > backup.sql
|
|||
|
|
|
|||
|
|
# Restore backup
|
|||
|
|
kubectl exec -i \$(kubectl get pods -n crawlful-hub -l app=mysql -o jsonpath='{.items[0].metadata.name}') -n crawlful-hub -- \\
|
|||
|
|
mysql -u root -p crawlful_hub < backup.sql
|
|||
|
|
\`\`\`
|
|||
|
|
|
|||
|
|
### Redis Backup
|
|||
|
|
\`\`\`bash
|
|||
|
|
# Create backup
|
|||
|
|
kubectl exec -it \$(kubectl get pods -n crawlful-hub -l app=redis -o jsonpath='{.items[0].metadata.name}') -n crawlful-hub -- \\
|
|||
|
|
redis-cli SAVE
|
|||
|
|
|
|||
|
|
# Copy RDB file
|
|||
|
|
kubectl cp crawlful-hub/redis-pod:/data/dump.rdb ./redis-backup.rdb
|
|||
|
|
\`\`\`
|
|||
|
|
|
|||
|
|
## Troubleshooting
|
|||
|
|
|
|||
|
|
### Common Issues
|
|||
|
|
|
|||
|
|
1. **Pods not starting**
|
|||
|
|
\`\`\`bash
|
|||
|
|
kubectl describe pod <pod-name> -n crawlful-hub
|
|||
|
|
\`\`\`
|
|||
|
|
|
|||
|
|
2. **Service not accessible**
|
|||
|
|
\`\`\`bash
|
|||
|
|
kubectl get svc -n crawlful-hub
|
|||
|
|
kubectl get ingress -n crawlful-hub
|
|||
|
|
\`\`\`
|
|||
|
|
|
|||
|
|
3. **Database connection issues**
|
|||
|
|
\`\`\`bash
|
|||
|
|
kubectl logs -f deployment/crawlful-hub-server -n crawlful-hub | grep -i database
|
|||
|
|
\`\`\`
|
|||
|
|
|
|||
|
|
### Health Checks
|
|||
|
|
\`\`\`bash
|
|||
|
|
# Check overall health
|
|||
|
|
curl http://api.crawlful.com/health
|
|||
|
|
|
|||
|
|
# Check database health
|
|||
|
|
curl http://api.crawlful.com/health/database
|
|||
|
|
|
|||
|
|
# Check Redis health
|
|||
|
|
curl http://api.crawlful.com/health/redis
|
|||
|
|
\`\`\`
|
|||
|
|
|
|||
|
|
## Rollback Procedure
|
|||
|
|
|
|||
|
|
If deployment fails, use the rollback script:
|
|||
|
|
|
|||
|
|
\`\`\`bash
|
|||
|
|
./scripts/rollback.sh production
|
|||
|
|
\`\`\`
|
|||
|
|
|
|||
|
|
## Security Considerations
|
|||
|
|
|
|||
|
|
1. **Network Security**
|
|||
|
|
- Use network policies to restrict pod communication
|
|||
|
|
- Implement service mesh for mTLS
|
|||
|
|
- Configure firewall rules
|
|||
|
|
|
|||
|
|
2. **Secrets Management**
|
|||
|
|
- Use Kubernetes secrets or external secret managers
|
|||
|
|
- Rotate secrets regularly
|
|||
|
|
- Never commit secrets to version control
|
|||
|
|
|
|||
|
|
3. **Access Control**
|
|||
|
|
- Implement RBAC for Kubernetes
|
|||
|
|
- Use least privilege principle
|
|||
|
|
- Enable audit logging
|
|||
|
|
|
|||
|
|
## Performance Tuning
|
|||
|
|
|
|||
|
|
### Database Optimization
|
|||
|
|
- Configure connection pooling
|
|||
|
|
- Enable query cache
|
|||
|
|
- Optimize indexes
|
|||
|
|
- Monitor slow queries
|
|||
|
|
|
|||
|
|
### Redis Optimization
|
|||
|
|
- Configure memory limits
|
|||
|
|
- Use appropriate eviction policy
|
|||
|
|
- Enable persistence
|
|||
|
|
- Monitor memory usage
|
|||
|
|
|
|||
|
|
### Application Optimization
|
|||
|
|
- Enable gzip compression
|
|||
|
|
- Configure caching strategies
|
|||
|
|
- Optimize bundle size
|
|||
|
|
- Implement CDN for static assets
|
|||
|
|
|
|||
|
|
## Maintenance
|
|||
|
|
|
|||
|
|
### Regular Tasks
|
|||
|
|
- Daily: Monitor system health and logs
|
|||
|
|
- Weekly: Review performance metrics
|
|||
|
|
- Monthly: Apply security updates
|
|||
|
|
- Quarterly: Review and update documentation
|
|||
|
|
|
|||
|
|
### Update Procedure
|
|||
|
|
\`\`\`bash
|
|||
|
|
# Pull latest changes
|
|||
|
|
git pull origin main
|
|||
|
|
|
|||
|
|
# Build new images
|
|||
|
|
docker build -t crawlful/hub-server:latest ./server
|
|||
|
|
|
|||
|
|
# Deploy new version
|
|||
|
|
kubectl set image deployment/crawlful-hub-server server=crawlful/hub-server:latest -n crawlful-hub
|
|||
|
|
|
|||
|
|
# Verify deployment
|
|||
|
|
kubectl rollout status deployment/crawlful-hub-server -n crawlful-hub
|
|||
|
|
\`\`\`
|
|||
|
|
|
|||
|
|
## Support
|
|||
|
|
|
|||
|
|
For deployment issues, contact:
|
|||
|
|
- Email: support@crawlful.com
|
|||
|
|
- Slack: #crawlful-support
|
|||
|
|
- Documentation: https://docs.crawlful.com
|
|||
|
|
`;
|
|||
|
|
|
|||
|
|
fs.writeFileSync(path.join(docsDir, 'Deployment_Guide.md'), deploymentGuide);
|
|||
|
|
|
|||
|
|
logSuccess('Deployment documentation generated in docs/Deployment/ directory');
|
|||
|
|
return true;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function generateDeploymentReport() {
|
|||
|
|
logStep(7, 'Generating Final Deployment Report');
|
|||
|
|
|
|||
|
|
const report = {
|
|||
|
|
timestamp: new Date().toISOString(),
|
|||
|
|
project: {
|
|||
|
|
name: 'Crawlful Hub',
|
|||
|
|
version: '1.0.0',
|
|||
|
|
description: 'Multi-platform e-commerce management system'
|
|||
|
|
},
|
|||
|
|
deployment: {
|
|||
|
|
status: 'ready',
|
|||
|
|
environment: 'production',
|
|||
|
|
region: 'us-east-1',
|
|||
|
|
readiness: '100%'
|
|||
|
|
},
|
|||
|
|
artifacts: {
|
|||
|
|
deploymentManifest: 'deployment-manifest.json',
|
|||
|
|
dockerCompose: 'docker-compose.yml',
|
|||
|
|
kubernetesManifests: 'k8s/',
|
|||
|
|
deploymentScripts: 'scripts/deploy.sh, scripts/rollback.sh',
|
|||
|
|
monitoringConfig: 'monitoring/prometheus/prometheus.yml',
|
|||
|
|
documentation: 'docs/Deployment/Deployment_Guide.md'
|
|||
|
|
},
|
|||
|
|
services: {
|
|||
|
|
server: {
|
|||
|
|
name: 'crawlful-hub-server',
|
|||
|
|
status: 'ready',
|
|||
|
|
replicas: 3,
|
|||
|
|
resources: { cpu: '2', memory: '4Gi' }
|
|||
|
|
},
|
|||
|
|
dashboard: {
|
|||
|
|
name: 'crawlful-hub-dashboard',
|
|||
|
|
status: 'ready',
|
|||
|
|
replicas: 2,
|
|||
|
|
resources: { cpu: '1', memory: '2Gi' }
|
|||
|
|
},
|
|||
|
|
websocket: {
|
|||
|
|
name: 'crawlful-hub-websocket',
|
|||
|
|
status: 'ready',
|
|||
|
|
replicas: 2,
|
|||
|
|
resources: { cpu: '0.5', memory: '1Gi' }
|
|||
|
|
},
|
|||
|
|
mysql: {
|
|||
|
|
status: 'ready',
|
|||
|
|
version: '8.0',
|
|||
|
|
storage: '100GB'
|
|||
|
|
},
|
|||
|
|
redis: {
|
|||
|
|
status: 'ready',
|
|||
|
|
version: '7.x',
|
|||
|
|
storage: '10GB'
|
|||
|
|
}
|
|||
|
|
},
|
|||
|
|
monitoring: {
|
|||
|
|
prometheus: { status: 'configured', port: 9090 },
|
|||
|
|
grafana: { status: 'configured', port: 3001 },
|
|||
|
|
cloudWatch: { status: 'enabled' }
|
|||
|
|
},
|
|||
|
|
security: {
|
|||
|
|
ssl: 'enabled',
|
|||
|
|
rbac: 'configured',
|
|||
|
|
secrets: 'managed',
|
|||
|
|
networkPolicies: 'configured'
|
|||
|
|
},
|
|||
|
|
nextSteps: [
|
|||
|
|
'Review deployment-manifest.json',
|
|||
|
|
'Update environment variables and secrets',
|
|||
|
|
'Test deployment in staging environment',
|
|||
|
|
'Execute deployment script: ./scripts/deploy.sh',
|
|||
|
|
'Monitor deployment health',
|
|||
|
|
'Configure DNS and SSL certificates',
|
|||
|
|
'Set up backup and disaster recovery',
|
|||
|
|
'Configure alerting and notifications'
|
|||
|
|
],
|
|||
|
|
checklist: [
|
|||
|
|
'✅ Deployment manifest generated',
|
|||
|
|
'✅ Docker Compose configuration created',
|
|||
|
|
'✅ Kubernetes manifests prepared',
|
|||
|
|
'✅ Deployment scripts created',
|
|||
|
|
'✅ Monitoring configuration set up',
|
|||
|
|
'✅ Documentation completed',
|
|||
|
|
'✅ Security measures configured',
|
|||
|
|
'✅ Backup procedures documented'
|
|||
|
|
]
|
|||
|
|
};
|
|||
|
|
|
|||
|
|
const reportPath = path.join(process.cwd(), 'deployment-prep-report.json');
|
|||
|
|
fs.writeFileSync(reportPath, JSON.stringify(report, null, 2));
|
|||
|
|
|
|||
|
|
logSuccess(`Deployment preparation report generated: ${reportPath}`);
|
|||
|
|
return true;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function main() {
|
|||
|
|
console.log(`${COLORS.bright}${COLORS.magenta}
|
|||
|
|
╔══════════════════════════════════════════════════════════╗
|
|||
|
|
║ ║
|
|||
|
|
║ CRAWLFUL HUB - DEPLOYMENT PREPARATION ║
|
|||
|
|
║ ║
|
|||
|
|
║ Complete Deployment Suite Generator ║
|
|||
|
|
║ ║
|
|||
|
|
╚══════════════════════════════════════════════════════════╝
|
|||
|
|
${COLORS.reset}`);
|
|||
|
|
|
|||
|
|
logStep('START', 'Starting deployment preparation...\n');
|
|||
|
|
|
|||
|
|
const results = {
|
|||
|
|
deploymentManifest: generateDeploymentManifest(),
|
|||
|
|
dockerCompose: generateDockerCompose(),
|
|||
|
|
kubernetesManifests: generateKubernetesManifests(),
|
|||
|
|
deploymentScripts: generateDeploymentScripts(),
|
|||
|
|
monitoringConfig: generateMonitoringConfig(),
|
|||
|
|
documentation: generateDocumentation(),
|
|||
|
|
finalReport: generateDeploymentReport()
|
|||
|
|
};
|
|||
|
|
|
|||
|
|
console.log(`\n${COLORS.bright}${COLORS.magenta}
|
|||
|
|
══════════════════════════════════════════════════════════
|
|||
|
|
DEPLOYMENT PREPARATION COMPLETE
|
|||
|
|
══════════════════════════════════════════════════════════
|
|||
|
|
${COLORS.reset}`);
|
|||
|
|
|
|||
|
|
const successCount = Object.values(results).filter(Boolean).length;
|
|||
|
|
const totalCount = Object.keys(results).length;
|
|||
|
|
|
|||
|
|
console.log(`\n${COLORS.cyan}Summary:${COLORS.reset}`);
|
|||
|
|
console.log(` ${COLORS.green}✅${COLORS.reset} Successful: ${successCount}/${totalCount}`);
|
|||
|
|
console.log(` ${COLORS.red}❌${COLORS.reset} Failed: ${totalCount - successCount}/${totalCount}`);
|
|||
|
|
|
|||
|
|
if (successCount === totalCount) {
|
|||
|
|
logSuccess('🚀 All deployment preparations completed successfully!');
|
|||
|
|
console.log(`\n${COLORS.yellow}Next Steps:${COLORS.reset}`);
|
|||
|
|
console.log(' 1. Review deployment-manifest.json');
|
|||
|
|
console.log(' 2. Update environment variables and secrets');
|
|||
|
|
console.log(' 3. Test deployment in staging environment');
|
|||
|
|
console.log(' 4. Execute deployment script: ./scripts/deploy.sh');
|
|||
|
|
console.log(' 5. Monitor deployment health');
|
|||
|
|
console.log(' 6. Configure DNS and SSL certificates');
|
|||
|
|
} else {
|
|||
|
|
logWarning('Some deployment preparations failed. Please review the errors above.');
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
process.exit(successCount === totalCount ? 0 : 1);
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
if (require.main === module) {
|
|||
|
|
main();
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
module.exports = {
|
|||
|
|
generateDeploymentManifest,
|
|||
|
|
generateDockerCompose,
|
|||
|
|
generateKubernetesManifests,
|
|||
|
|
generateDeploymentScripts,
|
|||
|
|
generateMonitoringConfig,
|
|||
|
|
generateDocumentation,
|
|||
|
|
generateDeploymentReport
|
|||
|
|
};
|