khondokartowsif171
511 lines
19 KiB
JavaScript
511 lines
19 KiB
JavaScript
const http = require('http');
|
||
const https = require('https');
|
||
const fs = require('fs');
|
||
const path = require('path');
|
||
const os = require('os');
|
||
const urlModule = require('url');
|
||
const crypto = require('crypto');
|
||
|
||
const PORT = 8080;
|
||
const DIRECTORY = __dirname;
|
||
const DATA_DIR = path.join(DIRECTORY, 'data');
|
||
const SYNCED_STORES_FILE = path.join(DATA_DIR, 'synced_stores.json');
|
||
|
||
if (!fs.existsSync(DATA_DIR)) {
|
||
fs.mkdirSync(DATA_DIR, { recursive: true });
|
||
}
|
||
if (!fs.existsSync(SYNCED_STORES_FILE)) {
|
||
fs.writeFileSync(SYNCED_STORES_FILE, JSON.stringify([], null, 2));
|
||
}
|
||
|
||
const MIME_TYPES = {
|
||
'.html': 'text/html; charset=UTF-8',
|
||
'.css': 'text/css',
|
||
'.js': 'application/javascript',
|
||
'.json': 'application/json',
|
||
'.png': 'image/png',
|
||
'.jpg': 'image/jpeg',
|
||
'.jpeg': 'image/jpeg',
|
||
'.svg': 'image/svg+xml',
|
||
'.ico': 'image/x-icon'
|
||
};
|
||
|
||
function getLocalIp() {
|
||
const interfaces = os.networkInterfaces();
|
||
for (const name of Object.keys(interfaces)) {
|
||
for (const iface of interfaces[name]) {
|
||
if (iface.family === 'IPv4' && !iface.internal) {
|
||
return iface.address;
|
||
}
|
||
}
|
||
}
|
||
return '127.0.0.1';
|
||
}
|
||
|
||
// Utility: Fetch URL with Redirects & Browser User-Agent
|
||
function fetchUrl(targetUrl, maxRedirects = 5) {
|
||
return new Promise((resolve, reject) => {
|
||
if (maxRedirects <= 0) return reject(new Error('Too many redirects'));
|
||
|
||
let parsed;
|
||
try {
|
||
parsed = new URL(targetUrl);
|
||
} catch (e) {
|
||
return reject(new Error(`Invalid URL: ${targetUrl}`));
|
||
}
|
||
|
||
const client = parsed.protocol === 'https:' ? https : http;
|
||
const req = client.get(targetUrl, {
|
||
headers: {
|
||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36',
|
||
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,application/json,image/webp,*/*;q=0.8',
|
||
'Accept-Language': 'en-US,en;q=0.9,bn;q=0.8'
|
||
},
|
||
timeout: 10000
|
||
}, (res) => {
|
||
if (res.statusCode >= 300 && res.statusCode < 400 && res.headers.location) {
|
||
const nextUrl = new URL(res.headers.location, targetUrl).href;
|
||
return fetchUrl(nextUrl, maxRedirects - 1).then(resolve).catch(reject);
|
||
}
|
||
|
||
let data = '';
|
||
res.setEncoding('utf8');
|
||
res.on('data', chunk => { data += chunk; });
|
||
res.on('end', () => {
|
||
resolve({
|
||
statusCode: res.statusCode,
|
||
headers: res.headers,
|
||
data: data,
|
||
finalUrl: targetUrl
|
||
});
|
||
});
|
||
});
|
||
|
||
req.on('timeout', () => {
|
||
req.destroy();
|
||
reject(new Error('Request timed out'));
|
||
});
|
||
req.on('error', reject);
|
||
});
|
||
}
|
||
|
||
// Helper: Convert English digits to Bengali format for prices
|
||
function toBengaliPrice(num) {
|
||
const bnDigits = ['০', '১', '২', '৩', '৪', '৫', '৬', '৭', '৮', '৯'];
|
||
const formattedNum = Math.round(num).toLocaleString('en-US');
|
||
const bn = formattedNum.replace(/\d/g, d => bnDigits[d]);
|
||
return `৳ ${bn}`;
|
||
}
|
||
|
||
// Helper: Recursively search nested JSON / state / props for product items
|
||
function findProductsInObject(obj, results, storeName, origin, category, storeId, depth = 0) {
|
||
if (!obj || depth > 8 || results.length >= 120) return;
|
||
|
||
if (Array.isArray(obj)) {
|
||
for (const item of obj) {
|
||
if (typeof item === 'object' && item !== null) {
|
||
const title = item.title || item.name || item.product_name || item.productTitle || '';
|
||
const price = item.price || item.sale_price || item.regular_price || item.unit_price || (item.variants && item.variants[0] ? item.variants[0].price : 0);
|
||
const image = item.image || item.thumbnail || item.featured_image || item.photo || (item.images && item.images[0] ? (item.images[0].src || item.images[0]) : '') || '';
|
||
|
||
if (title && typeof title === 'string' && title.length > 3 && (price || image)) {
|
||
const numPrice = parseFloat(String(price).replace(/[^0-9.]/g, '')) || 1200;
|
||
let imgUrl = typeof image === 'string' ? image : (image.url || image.src || '');
|
||
if (imgUrl && !imgUrl.startsWith('http') && !imgUrl.startsWith('data:')) {
|
||
try { imgUrl = new URL(imgUrl, origin).href; } catch(e) {}
|
||
}
|
||
|
||
results.push({
|
||
id: item.id ? String(item.id) : (item._id ? String(item._id) : crypto.randomUUID()),
|
||
title: title.trim(),
|
||
brand: item.brand?.name || item.brand || item.vendor || storeName,
|
||
price: toBengaliPrice(numPrice),
|
||
rawPrice: numPrice,
|
||
regularPrice: toBengaliPrice(numPrice * 1.25),
|
||
image: imgUrl,
|
||
link: item.url || (item.handle ? `${origin}/products/${item.handle}` : origin),
|
||
category: category,
|
||
storeName: storeName,
|
||
storeId: storeId
|
||
});
|
||
} else {
|
||
findProductsInObject(item, results, storeName, origin, category, storeId, depth + 1);
|
||
}
|
||
}
|
||
}
|
||
return;
|
||
}
|
||
|
||
if (typeof obj === 'object') {
|
||
for (const key of Object.keys(obj)) {
|
||
findProductsInObject(obj[key], results, storeName, origin, category, storeId, depth + 1);
|
||
}
|
||
}
|
||
}
|
||
|
||
// Multi-Strategy Real Store Scraper & Product Extractor
|
||
async function scrapeStore(rawUrl, customStoreName = '', customCategory = 'general') {
|
||
let targetUrl = rawUrl.trim();
|
||
if (!targetUrl.startsWith('http://') && !targetUrl.startsWith('https://')) {
|
||
targetUrl = 'https://' + targetUrl;
|
||
}
|
||
|
||
const parsedUrl = new URL(targetUrl);
|
||
const origin = parsedUrl.origin;
|
||
const hostname = parsedUrl.hostname.replace(/^www\./, '');
|
||
|
||
// Clean default store name from domain
|
||
const rawBrandFromDomain = hostname.split('.')[0];
|
||
const autoStoreName = customStoreName || (rawBrandFromDomain.charAt(0).toUpperCase() + rawBrandFromDomain.slice(1));
|
||
const storeId = 'store_' + hostname.replace(/[^a-zA-Z0-9]/g, '_').toLowerCase();
|
||
|
||
let products = [];
|
||
let detectedTitle = autoStoreName;
|
||
|
||
// STRATEGY 1: Check Shopify products.json API (extremely fast and structured)
|
||
try {
|
||
const shopifyUrl = `${origin}/products.json?limit=250`;
|
||
const shopifyRes = await fetchUrl(shopifyUrl);
|
||
if (shopifyRes.statusCode === 200) {
|
||
const shopifyData = JSON.parse(shopifyRes.data);
|
||
if (shopifyData && Array.isArray(shopifyData.products) && shopifyData.products.length > 0) {
|
||
shopifyData.products.forEach(p => {
|
||
const variant = p.variants && p.variants[0] ? p.variants[0] : {};
|
||
const priceVal = parseFloat(variant.price) || 1200;
|
||
const regPriceVal = parseFloat(variant.compare_at_price) || (priceVal * 1.22);
|
||
const imgUrl = (p.images && p.images[0] && p.images[0].src) ? p.images[0].src : ((p.image && p.image.src) ? p.image.src : '');
|
||
|
||
products.push({
|
||
id: p.id ? String(p.id) : crypto.randomUUID(),
|
||
title: p.title || 'Product',
|
||
brand: p.vendor || autoStoreName,
|
||
price: toBengaliPrice(priceVal),
|
||
rawPrice: priceVal,
|
||
regularPrice: toBengaliPrice(regPriceVal),
|
||
image: imgUrl,
|
||
link: `${origin}/products/${p.handle || p.id}`,
|
||
category: customCategory,
|
||
storeName: autoStoreName,
|
||
storeId: storeId
|
||
});
|
||
});
|
||
}
|
||
}
|
||
} catch (e) {
|
||
// Non-shopify or shopify API restricted, continue to HTML scraper
|
||
}
|
||
|
||
// STRATEGY 2: If products not found via Shopify API, fetch HTML and parse Schema.org JSON-LD & DOM
|
||
if (products.length === 0) {
|
||
const mainRes = await fetchUrl(targetUrl);
|
||
if (mainRes.statusCode !== 200) {
|
||
throw new Error(`Failed to fetch webpage (HTTP ${mainRes.statusCode})`);
|
||
}
|
||
|
||
const html = mainRes.data;
|
||
|
||
// Detect Page Title
|
||
const titleMatch = html.match(/<title[^>]*>([^<]+)<\/title>/i);
|
||
if (titleMatch && titleMatch[1]) {
|
||
const rawTitle = titleMatch[1].trim();
|
||
detectedTitle = customStoreName || rawTitle.split(/[|\-–—]/)[0].trim() || autoStoreName;
|
||
}
|
||
|
||
// 2A: Next.js __NEXT_DATA__ & Nuxt JSON State Extractor
|
||
const nextDataMatch = html.match(/<script[^>]+id=["']__NEXT_DATA__["'][^>]*>([\s\S]*?)<\/script>/i) ||
|
||
html.match(/<script[^>]*>window\.__INITIAL_STATE__\s*=\s*(\{[\s\S]*?\});<\/script>/i);
|
||
if (nextDataMatch && nextDataMatch[1]) {
|
||
try {
|
||
const stateJson = JSON.parse(nextDataMatch[1]);
|
||
findProductsInObject(stateJson, products, autoStoreName, origin, customCategory, storeId);
|
||
} catch (e) {}
|
||
}
|
||
|
||
// 2B: Schema.org JSON-LD Parsing
|
||
const jsonLdMatches = html.matchAll(/<script[^>]+type=["']application\/ld\+json["'][^>]*>([\s\S]*?)<\/script>/gi);
|
||
for (const match of jsonLdMatches) {
|
||
try {
|
||
const parsed = JSON.parse(match[1]);
|
||
const items = Array.isArray(parsed) ? parsed : (parsed['@graph'] || [parsed]);
|
||
|
||
for (const item of items) {
|
||
if (item['@type'] === 'Product') {
|
||
const priceVal = item.offers ? (parseFloat(item.offers.price || item.offers.lowPrice) || 1500) : 1500;
|
||
const regPriceVal = priceVal * 1.25;
|
||
const imgUrl = Array.isArray(item.image) ? item.image[0] : (item.image?.url || item.image || '');
|
||
|
||
products.push({
|
||
id: item.sku || crypto.randomUUID(),
|
||
title: item.name || 'Product',
|
||
brand: item.brand?.name || item.brand || autoStoreName,
|
||
price: toBengaliPrice(priceVal),
|
||
rawPrice: priceVal,
|
||
regularPrice: toBengaliPrice(regPriceVal),
|
||
image: typeof imgUrl === 'string' ? imgUrl : '',
|
||
link: item.url || targetUrl,
|
||
category: customCategory,
|
||
storeName: autoStoreName,
|
||
storeId: storeId
|
||
});
|
||
} else if (item['@type'] === 'ItemList' && Array.isArray(item.itemListElement)) {
|
||
for (const elem of item.itemListElement) {
|
||
const p = elem.item || elem;
|
||
if (p.name) {
|
||
const priceVal = p.offers ? (parseFloat(p.offers.price) || 1200) : 1200;
|
||
products.push({
|
||
id: crypto.randomUUID(),
|
||
title: p.name,
|
||
brand: p.brand?.name || autoStoreName,
|
||
price: toBengaliPrice(priceVal),
|
||
rawPrice: priceVal,
|
||
regularPrice: toBengaliPrice(priceVal * 1.25),
|
||
image: p.image || '',
|
||
link: p.url || targetUrl,
|
||
category: customCategory,
|
||
storeName: autoStoreName,
|
||
storeId: storeId
|
||
});
|
||
}
|
||
}
|
||
}
|
||
}
|
||
} catch (err) {
|
||
// Skip malformed JSON-LD block
|
||
}
|
||
}
|
||
|
||
// 2B: HTML Semantic Product Card Regex Parser (if JSON-LD had < 2 products)
|
||
if (products.length < 2) {
|
||
// Find product blocks
|
||
const cardRegex = /<(div|article|li)[^>]*class=["'][^"']*(product|item|grid|card)[^"']*["'][^>]*>([\s\S]*?)<\/\1>/gi;
|
||
let cardMatch;
|
||
let count = 0;
|
||
|
||
while ((cardMatch = cardRegex.exec(html)) !== null && count < 60) {
|
||
const block = cardMatch[3];
|
||
|
||
// Extract title
|
||
const titleTag = block.match(/<(h2|h3|h4|h5|a)[^>]*class=["'][^"']*(title|name|heading)[^"']*["'][^>]*>([^<]+)<\/\1>/i) ||
|
||
block.match(/<(h2|h3|h4|h5)[^>]*>([^<]+)<\/\1>/i) ||
|
||
block.match(/<a[^>]*title=["']([^"']+)["']/i);
|
||
const title = titleTag ? (titleTag[3] || titleTag[2] || titleTag[1]).trim() : '';
|
||
|
||
// Extract image
|
||
const imgTag = block.match(/<img[^>]+(?:src|data-src|data-lazy-src)=["']([^"']+)["'][^>]*>/i);
|
||
let imgUrl = imgTag ? imgTag[1] : '';
|
||
if (imgUrl && !imgUrl.startsWith('http') && !imgUrl.startsWith('data:')) {
|
||
imgUrl = new URL(imgUrl, origin).href;
|
||
}
|
||
|
||
// Extract link
|
||
const linkTag = block.match(/<a[^>]+href=["']([^"']+)["']/i);
|
||
let linkUrl = linkTag ? linkTag[1] : targetUrl;
|
||
if (linkUrl && !linkUrl.startsWith('http')) {
|
||
linkUrl = new URL(linkUrl, origin).href;
|
||
}
|
||
|
||
// Extract price
|
||
const priceTag = block.match(/(?:৳|Tk\.?|BDT|\$)\s*([\d,]+(?:\.\d+)?)/i) ||
|
||
block.match(/([\d,]+)\s*(?:৳|Tk\.?|BDT)/i);
|
||
let priceVal = 1450;
|
||
if (priceTag) {
|
||
const rawNum = parseFloat((priceTag[1] || priceTag[2] || '').replace(/,/g, ''));
|
||
if (rawNum && rawNum > 0) priceVal = rawNum;
|
||
}
|
||
|
||
if (title && title.length > 2 && (imgUrl || block.includes('price'))) {
|
||
count++;
|
||
products.push({
|
||
id: crypto.randomUUID(),
|
||
title: title.replace(/&/g, '&').replace(/'/g, "'"),
|
||
brand: autoStoreName,
|
||
price: toBengaliPrice(priceVal),
|
||
rawPrice: priceVal,
|
||
regularPrice: toBengaliPrice(priceVal * 1.25),
|
||
image: imgUrl,
|
||
link: linkUrl,
|
||
category: customCategory,
|
||
storeName: autoStoreName,
|
||
storeId: storeId
|
||
});
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
// Deduplicate products by title
|
||
const seenTitles = new Set();
|
||
products = products.filter(p => {
|
||
const key = p.title.toLowerCase().trim();
|
||
if (seenTitles.has(key)) return false;
|
||
seenTitles.add(key);
|
||
return true;
|
||
});
|
||
|
||
const finalStore = {
|
||
id: storeId,
|
||
name: autoStoreName,
|
||
detectedTitle: detectedTitle,
|
||
domain: hostname,
|
||
url: targetUrl,
|
||
category: customCategory,
|
||
syncedAt: new Date().toISOString(),
|
||
count: products.length,
|
||
products: products
|
||
};
|
||
|
||
// Save to persistent storage
|
||
let savedStores = [];
|
||
try {
|
||
if (fs.existsSync(SYNCED_STORES_FILE)) {
|
||
savedStores = JSON.parse(fs.readFileSync(SYNCED_STORES_FILE, 'utf8'));
|
||
}
|
||
} catch (e) {
|
||
savedStores = [];
|
||
}
|
||
|
||
// Upsert store
|
||
const existingIdx = savedStores.findIndex(s => s.id === storeId || s.domain === hostname);
|
||
if (existingIdx >= 0) {
|
||
savedStores[existingIdx] = finalStore;
|
||
} else {
|
||
savedStores.push(finalStore);
|
||
}
|
||
|
||
fs.writeFileSync(SYNCED_STORES_FILE, JSON.stringify(savedStores, null, 2), 'utf8');
|
||
|
||
return finalStore;
|
||
}
|
||
|
||
const server = http.createServer(async (req, res) => {
|
||
let reqPath = req.url.split('?')[0];
|
||
if (reqPath === '/' || reqPath === '') reqPath = '/index.html';
|
||
|
||
// API: Save Asset / Product Image
|
||
if (req.method === 'POST' && reqPath === '/api/save-asset') {
|
||
let body = '';
|
||
req.on('data', chunk => { body += chunk; });
|
||
req.on('end', () => {
|
||
try {
|
||
const { filename, base64Data } = JSON.parse(body);
|
||
const buffer = Buffer.from(base64Data.replace(/^data:image\/\w+;base64,/, ''), 'base64');
|
||
const savePath = path.join(DIRECTORY, 'assets', 'products', filename);
|
||
fs.writeFileSync(savePath, buffer);
|
||
res.writeHead(200, { 'Content-Type': 'application/json', 'Access-Control-Allow-Origin': '*' });
|
||
res.end(JSON.stringify({ success: true, savedTo: savePath }));
|
||
} catch (e) {
|
||
res.writeHead(500, { 'Content-Type': 'application/json', 'Access-Control-Allow-Origin': '*' });
|
||
res.end(JSON.stringify({ error: e.message }));
|
||
}
|
||
});
|
||
return;
|
||
}
|
||
|
||
// API: Universal Store URL Sync Endpoint
|
||
if (req.method === 'POST' && reqPath === '/api/sync-store') {
|
||
let body = '';
|
||
req.on('data', chunk => { body += chunk; });
|
||
req.on('end', async () => {
|
||
try {
|
||
const { url, storeName, category } = JSON.parse(body || '{}');
|
||
if (!url) {
|
||
res.writeHead(400, { 'Content-Type': 'application/json', 'Access-Control-Allow-Origin': '*' });
|
||
return res.end(JSON.stringify({ error: 'Store URL is required' }));
|
||
}
|
||
|
||
const storeResult = await scrapeStore(url, storeName, category);
|
||
res.writeHead(200, { 'Content-Type': 'application/json', 'Access-Control-Allow-Origin': '*' });
|
||
res.end(JSON.stringify({ success: true, store: storeResult }));
|
||
} catch (e) {
|
||
res.writeHead(500, { 'Content-Type': 'application/json', 'Access-Control-Allow-Origin': '*' });
|
||
res.end(JSON.stringify({ error: e.message }));
|
||
}
|
||
});
|
||
return;
|
||
}
|
||
|
||
// API: Get All Synced Stores
|
||
if (req.method === 'GET' && reqPath === '/api/get-synced-stores') {
|
||
try {
|
||
let savedStores = [];
|
||
if (fs.existsSync(SYNCED_STORES_FILE)) {
|
||
savedStores = JSON.parse(fs.readFileSync(SYNCED_STORES_FILE, 'utf8'));
|
||
}
|
||
res.writeHead(200, { 'Content-Type': 'application/json', 'Access-Control-Allow-Origin': '*' });
|
||
res.end(JSON.stringify({ success: true, stores: savedStores }));
|
||
} catch (e) {
|
||
res.writeHead(500, { 'Content-Type': 'application/json', 'Access-Control-Allow-Origin': '*' });
|
||
res.end(JSON.stringify({ error: e.message }));
|
||
}
|
||
return;
|
||
}
|
||
|
||
// API: Delete Synced Store
|
||
if (req.method === 'POST' && reqPath === '/api/delete-synced-store') {
|
||
let body = '';
|
||
req.on('data', chunk => { body += chunk; });
|
||
req.on('end', () => {
|
||
try {
|
||
const { storeId } = JSON.parse(body || '{}');
|
||
let savedStores = [];
|
||
if (fs.existsSync(SYNCED_STORES_FILE)) {
|
||
savedStores = JSON.parse(fs.readFileSync(SYNCED_STORES_FILE, 'utf8'));
|
||
}
|
||
savedStores = savedStores.filter(s => s.id !== storeId);
|
||
fs.writeFileSync(SYNCED_STORES_FILE, JSON.stringify(savedStores, null, 2), 'utf8');
|
||
res.writeHead(200, { 'Content-Type': 'application/json', 'Access-Control-Allow-Origin': '*' });
|
||
res.end(JSON.stringify({ success: true, stores: savedStores }));
|
||
} catch (e) {
|
||
res.writeHead(500, { 'Content-Type': 'application/json', 'Access-Control-Allow-Origin': '*' });
|
||
res.end(JSON.stringify({ error: e.message }));
|
||
}
|
||
});
|
||
return;
|
||
}
|
||
|
||
// CORS Preflight
|
||
if (req.method === 'OPTIONS') {
|
||
res.writeHead(204, {
|
||
'Access-Control-Allow-Origin': '*',
|
||
'Access-Control-Allow-Methods': 'GET, POST, OPTIONS',
|
||
'Access-Control-Allow-Headers': 'Content-Type'
|
||
});
|
||
return res.end();
|
||
}
|
||
|
||
const filePath = path.join(DIRECTORY, reqPath);
|
||
const ext = path.extname(filePath).toLowerCase();
|
||
const contentType = MIME_TYPES[ext] || 'application/octet-stream';
|
||
|
||
fs.readFile(filePath, (err, content) => {
|
||
if (err) {
|
||
if (err.code === 'ENOENT') {
|
||
res.writeHead(404, { 'Content-Type': 'text/plain' });
|
||
res.end('404 Not Found');
|
||
} else {
|
||
res.writeHead(500, { 'Content-Type': 'text/plain' });
|
||
res.end(`Server Error: ${err.code}`);
|
||
}
|
||
} else {
|
||
res.writeHead(200, {
|
||
'Content-Type': contentType,
|
||
'Access-Control-Allow-Origin': '*'
|
||
});
|
||
res.end(content);
|
||
}
|
||
});
|
||
});
|
||
|
||
server.listen(PORT, '0.0.0.0', () => {
|
||
const localIp = getLocalIp();
|
||
console.log('='.repeat(60));
|
||
console.log('💎 Google Opal AI - iOS Web Studio is Live & Running!');
|
||
console.log('='.repeat(60));
|
||
console.log(`📱 Open on your iPhone Safari (connected to same Wi-Fi):`);
|
||
console.log(` 👉 http://${localIp}:${PORT}`);
|
||
console.log(`💻 Or on PC Browser: http://localhost:${PORT}`);
|
||
console.log('='.repeat(60));
|
||
console.log('💡 iPhone Instruction: Safari-তে গিয়ে "Share ➔ Add to Home Screen" দিন।');
|
||
console.log('='.repeat(60));
|
||
});
|