const http = require('http'); const https = require('https'); const fs = require('fs'); const path = require('path'); const os = require('os'); const urlModule = require('url'); const crypto = require('crypto'); const PORT = 8080; const DIRECTORY = __dirname; const DATA_DIR = path.join(DIRECTORY, 'data'); const SYNCED_STORES_FILE = path.join(DATA_DIR, 'synced_stores.json'); if (!fs.existsSync(DATA_DIR)) { fs.mkdirSync(DATA_DIR, { recursive: true }); } if (!fs.existsSync(SYNCED_STORES_FILE)) { fs.writeFileSync(SYNCED_STORES_FILE, JSON.stringify([], null, 2)); } const MIME_TYPES = { '.html': 'text/html; charset=UTF-8', '.css': 'text/css', '.js': 'application/javascript', '.json': 'application/json', '.png': 'image/png', '.jpg': 'image/jpeg', '.jpeg': 'image/jpeg', '.svg': 'image/svg+xml', '.ico': 'image/x-icon' }; function getLocalIp() { const interfaces = os.networkInterfaces(); for (const name of Object.keys(interfaces)) { for (const iface of interfaces[name]) { if (iface.family === 'IPv4' && !iface.internal) { return iface.address; } } } return '127.0.0.1'; } // Utility: Fetch URL with Redirects & Browser User-Agent function fetchUrl(targetUrl, maxRedirects = 5) { return new Promise((resolve, reject) => { if (maxRedirects <= 0) return reject(new Error('Too many redirects')); let parsed; try { parsed = new URL(targetUrl); } catch (e) { return reject(new Error(`Invalid URL: ${targetUrl}`)); } const client = parsed.protocol === 'https:' ? https : http; const req = client.get(targetUrl, { headers: { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36', 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,application/json,image/webp,*/*;q=0.8', 'Accept-Language': 'en-US,en;q=0.9,bn;q=0.8' }, timeout: 10000 }, (res) => { if (res.statusCode >= 300 && res.statusCode < 400 && res.headers.location) { const nextUrl = new URL(res.headers.location, targetUrl).href; return fetchUrl(nextUrl, maxRedirects - 1).then(resolve).catch(reject); } let data = ''; res.setEncoding('utf8'); res.on('data', chunk => { data += chunk; }); res.on('end', () => { resolve({ statusCode: res.statusCode, headers: res.headers, data: data, finalUrl: targetUrl }); }); }); req.on('timeout', () => { req.destroy(); reject(new Error('Request timed out')); }); req.on('error', reject); }); } // Helper: Convert English digits to Bengali format for prices function toBengaliPrice(num) { const bnDigits = ['০', '১', '২', '৩', '৪', '৫', '৬', '৭', '৮', '৯']; const formattedNum = Math.round(num).toLocaleString('en-US'); const bn = formattedNum.replace(/\d/g, d => bnDigits[d]); return `৳ ${bn}`; } // Helper: Recursively search nested JSON / state / props for product items function findProductsInObject(obj, results, storeName, origin, category, storeId, depth = 0) { if (!obj || depth > 8 || results.length >= 120) return; if (Array.isArray(obj)) { for (const item of obj) { if (typeof item === 'object' && item !== null) { const title = item.title || item.name || item.product_name || item.productTitle || ''; const price = item.price || item.sale_price || item.regular_price || item.unit_price || (item.variants && item.variants[0] ? item.variants[0].price : 0); const image = item.image || item.thumbnail || item.featured_image || item.photo || (item.images && item.images[0] ? (item.images[0].src || item.images[0]) : '') || ''; if (title && typeof title === 'string' && title.length > 3 && (price || image)) { const numPrice = parseFloat(String(price).replace(/[^0-9.]/g, '')) || 1200; let imgUrl = typeof image === 'string' ? image : (image.url || image.src || ''); if (imgUrl && !imgUrl.startsWith('http') && !imgUrl.startsWith('data:')) { try { imgUrl = new URL(imgUrl, origin).href; } catch(e) {} } results.push({ id: item.id ? String(item.id) : (item._id ? String(item._id) : crypto.randomUUID()), title: title.trim(), brand: item.brand?.name || item.brand || item.vendor || storeName, price: toBengaliPrice(numPrice), rawPrice: numPrice, regularPrice: toBengaliPrice(numPrice * 1.25), image: imgUrl, link: item.url || (item.handle ? `${origin}/products/${item.handle}` : origin), category: category, storeName: storeName, storeId: storeId }); } else { findProductsInObject(item, results, storeName, origin, category, storeId, depth + 1); } } } return; } if (typeof obj === 'object') { for (const key of Object.keys(obj)) { findProductsInObject(obj[key], results, storeName, origin, category, storeId, depth + 1); } } } // Multi-Strategy Real Store Scraper & Product Extractor async function scrapeStore(rawUrl, customStoreName = '', customCategory = 'general') { let targetUrl = rawUrl.trim(); if (!targetUrl.startsWith('http://') && !targetUrl.startsWith('https://')) { targetUrl = 'https://' + targetUrl; } const parsedUrl = new URL(targetUrl); const origin = parsedUrl.origin; const hostname = parsedUrl.hostname.replace(/^www\./, ''); // Clean default store name from domain const rawBrandFromDomain = hostname.split('.')[0]; const autoStoreName = customStoreName || (rawBrandFromDomain.charAt(0).toUpperCase() + rawBrandFromDomain.slice(1)); const storeId = 'store_' + hostname.replace(/[^a-zA-Z0-9]/g, '_').toLowerCase(); let products = []; let detectedTitle = autoStoreName; // STRATEGY 1: Check Shopify products.json API (extremely fast and structured) try { const shopifyUrl = `${origin}/products.json?limit=250`; const shopifyRes = await fetchUrl(shopifyUrl); if (shopifyRes.statusCode === 200) { const shopifyData = JSON.parse(shopifyRes.data); if (shopifyData && Array.isArray(shopifyData.products) && shopifyData.products.length > 0) { shopifyData.products.forEach(p => { const variant = p.variants && p.variants[0] ? p.variants[0] : {}; const priceVal = parseFloat(variant.price) || 1200; const regPriceVal = parseFloat(variant.compare_at_price) || (priceVal * 1.22); const imgUrl = (p.images && p.images[0] && p.images[0].src) ? p.images[0].src : ((p.image && p.image.src) ? p.image.src : ''); products.push({ id: p.id ? String(p.id) : crypto.randomUUID(), title: p.title || 'Product', brand: p.vendor || autoStoreName, price: toBengaliPrice(priceVal), rawPrice: priceVal, regularPrice: toBengaliPrice(regPriceVal), image: imgUrl, link: `${origin}/products/${p.handle || p.id}`, category: customCategory, storeName: autoStoreName, storeId: storeId }); }); } } } catch (e) { // Non-shopify or shopify API restricted, continue to HTML scraper } // STRATEGY 2: If products not found via Shopify API, fetch HTML and parse Schema.org JSON-LD & DOM if (products.length === 0) { const mainRes = await fetchUrl(targetUrl); if (mainRes.statusCode !== 200) { throw new Error(`Failed to fetch webpage (HTTP ${mainRes.statusCode})`); } const html = mainRes.data; // Detect Page Title const titleMatch = html.match(/]*>([^<]+)<\/title>/i); if (titleMatch && titleMatch[1]) { const rawTitle = titleMatch[1].trim(); detectedTitle = customStoreName || rawTitle.split(/[|\-–—]/)[0].trim() || autoStoreName; } // 2A: Next.js __NEXT_DATA__ & Nuxt JSON State Extractor const nextDataMatch = html.match(/]+id=["']__NEXT_DATA__["'][^>]*>([\s\S]*?)<\/script>/i) || html.match(/]*>window\.__INITIAL_STATE__\s*=\s*(\{[\s\S]*?\});<\/script>/i); if (nextDataMatch && nextDataMatch[1]) { try { const stateJson = JSON.parse(nextDataMatch[1]); findProductsInObject(stateJson, products, autoStoreName, origin, customCategory, storeId); } catch (e) {} } // 2B: Schema.org JSON-LD Parsing const jsonLdMatches = html.matchAll(/]+type=["']application\/ld\+json["'][^>]*>([\s\S]*?)<\/script>/gi); for (const match of jsonLdMatches) { try { const parsed = JSON.parse(match[1]); const items = Array.isArray(parsed) ? parsed : (parsed['@graph'] || [parsed]); for (const item of items) { if (item['@type'] === 'Product') { const priceVal = item.offers ? (parseFloat(item.offers.price || item.offers.lowPrice) || 1500) : 1500; const regPriceVal = priceVal * 1.25; const imgUrl = Array.isArray(item.image) ? item.image[0] : (item.image?.url || item.image || ''); products.push({ id: item.sku || crypto.randomUUID(), title: item.name || 'Product', brand: item.brand?.name || item.brand || autoStoreName, price: toBengaliPrice(priceVal), rawPrice: priceVal, regularPrice: toBengaliPrice(regPriceVal), image: typeof imgUrl === 'string' ? imgUrl : '', link: item.url || targetUrl, category: customCategory, storeName: autoStoreName, storeId: storeId }); } else if (item['@type'] === 'ItemList' && Array.isArray(item.itemListElement)) { for (const elem of item.itemListElement) { const p = elem.item || elem; if (p.name) { const priceVal = p.offers ? (parseFloat(p.offers.price) || 1200) : 1200; products.push({ id: crypto.randomUUID(), title: p.name, brand: p.brand?.name || autoStoreName, price: toBengaliPrice(priceVal), rawPrice: priceVal, regularPrice: toBengaliPrice(priceVal * 1.25), image: p.image || '', link: p.url || targetUrl, category: customCategory, storeName: autoStoreName, storeId: storeId }); } } } } } catch (err) { // Skip malformed JSON-LD block } } // 2B: HTML Semantic Product Card Regex Parser (if JSON-LD had < 2 products) if (products.length < 2) { // Find product blocks const cardRegex = /<(div|article|li)[^>]*class=["'][^"']*(product|item|grid|card)[^"']*["'][^>]*>([\s\S]*?)<\/\1>/gi; let cardMatch; let count = 0; while ((cardMatch = cardRegex.exec(html)) !== null && count < 60) { const block = cardMatch[3]; // Extract title const titleTag = block.match(/<(h2|h3|h4|h5|a)[^>]*class=["'][^"']*(title|name|heading)[^"']*["'][^>]*>([^<]+)<\/\1>/i) || block.match(/<(h2|h3|h4|h5)[^>]*>([^<]+)<\/\1>/i) || block.match(/]*title=["']([^"']+)["']/i); const title = titleTag ? (titleTag[3] || titleTag[2] || titleTag[1]).trim() : ''; // Extract image const imgTag = block.match(/]+(?:src|data-src|data-lazy-src)=["']([^"']+)["'][^>]*>/i); let imgUrl = imgTag ? imgTag[1] : ''; if (imgUrl && !imgUrl.startsWith('http') && !imgUrl.startsWith('data:')) { imgUrl = new URL(imgUrl, origin).href; } // Extract link const linkTag = block.match(/]+href=["']([^"']+)["']/i); let linkUrl = linkTag ? linkTag[1] : targetUrl; if (linkUrl && !linkUrl.startsWith('http')) { linkUrl = new URL(linkUrl, origin).href; } // Extract price const priceTag = block.match(/(?:৳|Tk\.?|BDT|\$)\s*([\d,]+(?:\.\d+)?)/i) || block.match(/([\d,]+)\s*(?:৳|Tk\.?|BDT)/i); let priceVal = 1450; if (priceTag) { const rawNum = parseFloat((priceTag[1] || priceTag[2] || '').replace(/,/g, '')); if (rawNum && rawNum > 0) priceVal = rawNum; } if (title && title.length > 2 && (imgUrl || block.includes('price'))) { count++; products.push({ id: crypto.randomUUID(), title: title.replace(/&/g, '&').replace(/'/g, "'"), brand: autoStoreName, price: toBengaliPrice(priceVal), rawPrice: priceVal, regularPrice: toBengaliPrice(priceVal * 1.25), image: imgUrl, link: linkUrl, category: customCategory, storeName: autoStoreName, storeId: storeId }); } } } } // Deduplicate products by title const seenTitles = new Set(); products = products.filter(p => { const key = p.title.toLowerCase().trim(); if (seenTitles.has(key)) return false; seenTitles.add(key); return true; }); const finalStore = { id: storeId, name: autoStoreName, detectedTitle: detectedTitle, domain: hostname, url: targetUrl, category: customCategory, syncedAt: new Date().toISOString(), count: products.length, products: products }; // Save to persistent storage let savedStores = []; try { if (fs.existsSync(SYNCED_STORES_FILE)) { savedStores = JSON.parse(fs.readFileSync(SYNCED_STORES_FILE, 'utf8')); } } catch (e) { savedStores = []; } // Upsert store const existingIdx = savedStores.findIndex(s => s.id === storeId || s.domain === hostname); if (existingIdx >= 0) { savedStores[existingIdx] = finalStore; } else { savedStores.push(finalStore); } fs.writeFileSync(SYNCED_STORES_FILE, JSON.stringify(savedStores, null, 2), 'utf8'); return finalStore; } const server = http.createServer(async (req, res) => { let reqPath = req.url.split('?')[0]; if (reqPath === '/' || reqPath === '') reqPath = '/index.html'; // API: Save Asset / Product Image if (req.method === 'POST' && reqPath === '/api/save-asset') { let body = ''; req.on('data', chunk => { body += chunk; }); req.on('end', () => { try { const { filename, base64Data } = JSON.parse(body); const buffer = Buffer.from(base64Data.replace(/^data:image\/\w+;base64,/, ''), 'base64'); const savePath = path.join(DIRECTORY, 'assets', 'products', filename); fs.writeFileSync(savePath, buffer); res.writeHead(200, { 'Content-Type': 'application/json', 'Access-Control-Allow-Origin': '*' }); res.end(JSON.stringify({ success: true, savedTo: savePath })); } catch (e) { res.writeHead(500, { 'Content-Type': 'application/json', 'Access-Control-Allow-Origin': '*' }); res.end(JSON.stringify({ error: e.message })); } }); return; } // API: Universal Store URL Sync Endpoint if (req.method === 'POST' && reqPath === '/api/sync-store') { let body = ''; req.on('data', chunk => { body += chunk; }); req.on('end', async () => { try { const { url, storeName, category } = JSON.parse(body || '{}'); if (!url) { res.writeHead(400, { 'Content-Type': 'application/json', 'Access-Control-Allow-Origin': '*' }); return res.end(JSON.stringify({ error: 'Store URL is required' })); } const storeResult = await scrapeStore(url, storeName, category); res.writeHead(200, { 'Content-Type': 'application/json', 'Access-Control-Allow-Origin': '*' }); res.end(JSON.stringify({ success: true, store: storeResult })); } catch (e) { res.writeHead(500, { 'Content-Type': 'application/json', 'Access-Control-Allow-Origin': '*' }); res.end(JSON.stringify({ error: e.message })); } }); return; } // API: Get All Synced Stores if (req.method === 'GET' && reqPath === '/api/get-synced-stores') { try { let savedStores = []; if (fs.existsSync(SYNCED_STORES_FILE)) { savedStores = JSON.parse(fs.readFileSync(SYNCED_STORES_FILE, 'utf8')); } res.writeHead(200, { 'Content-Type': 'application/json', 'Access-Control-Allow-Origin': '*' }); res.end(JSON.stringify({ success: true, stores: savedStores })); } catch (e) { res.writeHead(500, { 'Content-Type': 'application/json', 'Access-Control-Allow-Origin': '*' }); res.end(JSON.stringify({ error: e.message })); } return; } // API: Delete Synced Store if (req.method === 'POST' && reqPath === '/api/delete-synced-store') { let body = ''; req.on('data', chunk => { body += chunk; }); req.on('end', () => { try { const { storeId } = JSON.parse(body || '{}'); let savedStores = []; if (fs.existsSync(SYNCED_STORES_FILE)) { savedStores = JSON.parse(fs.readFileSync(SYNCED_STORES_FILE, 'utf8')); } savedStores = savedStores.filter(s => s.id !== storeId); fs.writeFileSync(SYNCED_STORES_FILE, JSON.stringify(savedStores, null, 2), 'utf8'); res.writeHead(200, { 'Content-Type': 'application/json', 'Access-Control-Allow-Origin': '*' }); res.end(JSON.stringify({ success: true, stores: savedStores })); } catch (e) { res.writeHead(500, { 'Content-Type': 'application/json', 'Access-Control-Allow-Origin': '*' }); res.end(JSON.stringify({ error: e.message })); } }); return; } // CORS Preflight if (req.method === 'OPTIONS') { res.writeHead(204, { 'Access-Control-Allow-Origin': '*', 'Access-Control-Allow-Methods': 'GET, POST, OPTIONS', 'Access-Control-Allow-Headers': 'Content-Type' }); return res.end(); } const filePath = path.join(DIRECTORY, reqPath); const ext = path.extname(filePath).toLowerCase(); const contentType = MIME_TYPES[ext] || 'application/octet-stream'; fs.readFile(filePath, (err, content) => { if (err) { if (err.code === 'ENOENT') { res.writeHead(404, { 'Content-Type': 'text/plain' }); res.end('404 Not Found'); } else { res.writeHead(500, { 'Content-Type': 'text/plain' }); res.end(`Server Error: ${err.code}`); } } else { res.writeHead(200, { 'Content-Type': contentType, 'Access-Control-Allow-Origin': '*' }); res.end(content); } }); }); server.listen(PORT, '0.0.0.0', () => { const localIp = getLocalIp(); console.log('='.repeat(60)); console.log('💎 Google Opal AI - iOS Web Studio is Live & Running!'); console.log('='.repeat(60)); console.log(`📱 Open on your iPhone Safari (connected to same Wi-Fi):`); console.log(` 👉 http://${localIp}:${PORT}`); console.log(`💻 Or on PC Browser: http://localhost:${PORT}`); console.log('='.repeat(60)); console.log('💡 iPhone Instruction: Safari-তে গিয়ে "Share ➔ Add to Home Screen" দিন।'); console.log('='.repeat(60)); });