const http = require('http'); const https = require('https'); const fs = require('fs'); const path = require('path'); const os = require('os'); const urlModule = require('url'); const crypto = require('crypto'); const PORT = 8080; const DIRECTORY = __dirname; const DATA_DIR = path.join(DIRECTORY, 'data'); const SYNCED_STORES_FILE = path.join(DATA_DIR, 'synced_stores.json'); if (!fs.existsSync(DATA_DIR)) { fs.mkdirSync(DATA_DIR, { recursive: true }); } if (!fs.existsSync(SYNCED_STORES_FILE)) { fs.writeFileSync(SYNCED_STORES_FILE, JSON.stringify([], null, 2)); } const MIME_TYPES = { '.html': 'text/html; charset=UTF-8', '.css': 'text/css', '.js': 'application/javascript', '.json': 'application/json', '.png': 'image/png', '.jpg': 'image/jpeg', '.jpeg': 'image/jpeg', '.svg': 'image/svg+xml', '.ico': 'image/x-icon' }; function getLocalIp() { const interfaces = os.networkInterfaces(); for (const name of Object.keys(interfaces)) { for (const iface of interfaces[name]) { if (iface.family === 'IPv4' && !iface.internal) { return iface.address; } } } return '127.0.0.1'; } // Utility: Fetch URL with Redirects & Browser User-Agent function fetchUrl(targetUrl, maxRedirects = 5) { return new Promise((resolve, reject) => { if (maxRedirects <= 0) return reject(new Error('Too many redirects')); let parsed; try { parsed = new URL(targetUrl); } catch (e) { return reject(new Error(`Invalid URL: ${targetUrl}`)); } const client = parsed.protocol === 'https:' ? https : http; const req = client.get(targetUrl, { headers: { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36', 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,application/json,image/webp,*/*;q=0.8', 'Accept-Language': 'en-US,en;q=0.9,bn;q=0.8' }, timeout: 10000 }, (res) => { if (res.statusCode >= 300 && res.statusCode < 400 && res.headers.location) { const nextUrl = new URL(res.headers.location, targetUrl).href; return fetchUrl(nextUrl, maxRedirects - 1).then(resolve).catch(reject); } let data = ''; res.setEncoding('utf8'); res.on('data', chunk => { data += chunk; }); res.on('end', () => { resolve({ statusCode: res.statusCode, headers: res.headers, data: data, finalUrl: targetUrl }); }); }); req.on('timeout', () => { req.destroy(); reject(new Error('Request timed out')); }); req.on('error', reject); }); } // Helper: Convert English digits to Bengali format for prices function toBengaliPrice(num) { const bnDigits = ['০', '১', '২', '৩', '৪', '৫', '৬', '৭', '৮', '৯']; const formattedNum = Math.round(num).toLocaleString('en-US'); const bn = formattedNum.replace(/\d/g, d => bnDigits[d]); return `৳ ${bn}`; } // Helper: Recursively search nested JSON / state / props for product items function findProductsInObject(obj, results, storeName, origin, category, storeId, depth = 0) { if (!obj || depth > 8 || results.length >= 120) return; if (Array.isArray(obj)) { for (const item of obj) { if (typeof item === 'object' && item !== null) { const title = item.title || item.name || item.product_name || item.productTitle || ''; const price = item.price || item.sale_price || item.regular_price || item.unit_price || (item.variants && item.variants[0] ? item.variants[0].price : 0); const image = item.image || item.thumbnail || item.featured_image || item.photo || (item.images && item.images[0] ? (item.images[0].src || item.images[0]) : '') || ''; if (title && typeof title === 'string' && title.length > 3 && (price || image)) { const numPrice = parseFloat(String(price).replace(/[^0-9.]/g, '')) || 1200; let imgUrl = typeof image === 'string' ? image : (image.url || image.src || ''); if (imgUrl && !imgUrl.startsWith('http') && !imgUrl.startsWith('data:')) { try { imgUrl = new URL(imgUrl, origin).href; } catch(e) {} } results.push({ id: item.id ? String(item.id) : (item._id ? String(item._id) : crypto.randomUUID()), title: title.trim(), brand: item.brand?.name || item.brand || item.vendor || storeName, price: toBengaliPrice(numPrice), rawPrice: numPrice, regularPrice: toBengaliPrice(numPrice * 1.25), image: imgUrl, link: item.url || (item.handle ? `${origin}/products/${item.handle}` : origin), category: category, storeName: storeName, storeId: storeId }); } else { findProductsInObject(item, results, storeName, origin, category, storeId, depth + 1); } } } return; } if (typeof obj === 'object') { for (const key of Object.keys(obj)) { findProductsInObject(obj[key], results, storeName, origin, category, storeId, depth + 1); } } } // Multi-Strategy Real Store Scraper & Product Extractor async function scrapeStore(rawUrl, customStoreName = '', customCategory = 'general') { let targetUrl = rawUrl.trim(); if (!targetUrl.startsWith('http://') && !targetUrl.startsWith('https://')) { targetUrl = 'https://' + targetUrl; } const parsedUrl = new URL(targetUrl); const origin = parsedUrl.origin; const hostname = parsedUrl.hostname.replace(/^www\./, ''); // Clean default store name from domain const rawBrandFromDomain = hostname.split('.')[0]; const autoStoreName = customStoreName || (rawBrandFromDomain.charAt(0).toUpperCase() + rawBrandFromDomain.slice(1)); const storeId = 'store_' + hostname.replace(/[^a-zA-Z0-9]/g, '_').toLowerCase(); let products = []; let detectedTitle = autoStoreName; // STRATEGY 1: Check Shopify products.json API (extremely fast and structured) try { const shopifyUrl = `${origin}/products.json?limit=250`; const shopifyRes = await fetchUrl(shopifyUrl); if (shopifyRes.statusCode === 200) { const shopifyData = JSON.parse(shopifyRes.data); if (shopifyData && Array.isArray(shopifyData.products) && shopifyData.products.length > 0) { shopifyData.products.forEach(p => { const variant = p.variants && p.variants[0] ? p.variants[0] : {}; const priceVal = parseFloat(variant.price) || 1200; const regPriceVal = parseFloat(variant.compare_at_price) || (priceVal * 1.22); const imgUrl = (p.images && p.images[0] && p.images[0].src) ? p.images[0].src : ((p.image && p.image.src) ? p.image.src : ''); products.push({ id: p.id ? String(p.id) : crypto.randomUUID(), title: p.title || 'Product', brand: p.vendor || autoStoreName, price: toBengaliPrice(priceVal), rawPrice: priceVal, regularPrice: toBengaliPrice(regPriceVal), image: imgUrl, link: `${origin}/products/${p.handle || p.id}`, category: customCategory, storeName: autoStoreName, storeId: storeId }); }); } } } catch (e) { // Non-shopify or shopify API restricted, continue to HTML scraper } // STRATEGY 2: If products not found via Shopify API, fetch HTML and parse Schema.org JSON-LD & DOM if (products.length === 0) { const mainRes = await fetchUrl(targetUrl); if (mainRes.statusCode !== 200) { throw new Error(`Failed to fetch webpage (HTTP ${mainRes.statusCode})`); } const html = mainRes.data; // Detect Page Title const titleMatch = html.match(/