609 lines
21 KiB
HTML
609 lines
21 KiB
HTML
<!DOCTYPE html>
|
|
<html lang="en">
|
|
<head>
|
|
<meta charset="UTF-8">
|
|
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
|
<title>Network Visualization</title>
|
|
<script src="https://d3js.org/d3.v6.min.js"></script>
|
|
<style>
|
|
body, html {
|
|
margin: 0;
|
|
padding: 0;
|
|
width: 100%;
|
|
height: 100%;
|
|
overflow: hidden;
|
|
}
|
|
canvas {
|
|
width: 100%;
|
|
height: 100%;
|
|
background: #000;
|
|
}
|
|
|
|
#legend li {
|
|
margin: 5px 0;
|
|
display: flex;
|
|
align-items: center;
|
|
}
|
|
|
|
.legend-color-box {
|
|
width: 20px;
|
|
height: 20px;
|
|
margin-right: 5px;
|
|
border: 1px solid #ddd;
|
|
}
|
|
</style>
|
|
</head>
|
|
<body>
|
|
<canvas id="networkCanvas"></canvas>
|
|
|
|
<div id="controls-container" style="position: absolute; bottom: 80px; left: 10px;">
|
|
<button id="clear-db-button" style="color: white; background-color: #333; border: none; padding: 5px 10px; cursor: pointer; margin-bottom: 10px;">Clear Database</button>
|
|
<button id="toggle-simulation" style="color: white; background-color: #333; border: none; padding: 5px 10px; cursor: pointer;">Pause</button>
|
|
</div>
|
|
|
|
<div id="slider-container" style="position: absolute; bottom: 10px; left: 10px; text-align: center;">
|
|
<label for="scale-slider" style="color: white; display: block; margin-bottom: 5px;">Link Distance Scale:</label>
|
|
<input type="range" id="scale-slider" min="1" max="10" step="0.1" value="1" style="color: white;">
|
|
</div>
|
|
|
|
<div id="search-container" style="position: absolute; top: 10px; left: 50%; transform: translateX(-50%); text-align: center;">
|
|
<div style="display: inline-block; text-align: left;">
|
|
<label for="url-input" style="color: white; display: block; margin-bottom: 5px;">Enter URL Here:</label>
|
|
<input type="text" id="url-input" style="color: black; padding: 5px; display: block;">
|
|
</div>
|
|
|
|
<div style="display: inline-block; text-align: left; margin-left: 10px;">
|
|
<label for="depth-input" style="color: white; display: block; margin-bottom: 5px;">Max Depth:</label>
|
|
<input type="number" id="depth-input" min="1" max="10" step="1" value="4" style="color: black; padding: 5px; display: block;">
|
|
</div>
|
|
|
|
<button id="search-button" style="color: white; background-color: #333; border: none; padding: 5px 10px; cursor: pointer; margin-top: 10px;">Search</button>
|
|
</div>
|
|
|
|
<div id="legend-container" style="position: absolute; top: 10px; right: 10px; background: rgba(0,0,0,0.8); padding: 10px; border-radius: 5px; border: 1px solid #ddd; color: white;">
|
|
<h3 style="color: white; margin-top: 0;">Color Legend</h3>
|
|
<ul id="legend" style="list-style: none; padding: 0;">
|
|
<!-- Color tags will be added here dynamically -->
|
|
</ul>
|
|
</div>
|
|
|
|
|
|
|
|
<script src="https://cdnjs.cloudflare.com/ajax/libs/three.js/r128/three.min.js"></script>
|
|
<script>
|
|
const canvas = document.getElementById('networkCanvas');
|
|
const ctx = canvas.getContext('2d');
|
|
canvas.width = window.innerWidth;
|
|
canvas.height = window.innerHeight;
|
|
|
|
const urlInput = document.getElementById('url-input');
|
|
const depthInput = document.getElementById('depth-input');
|
|
const searchButton = document.getElementById('search-button');
|
|
|
|
document.getElementById('clear-db-button').addEventListener('click', function() {
|
|
if (confirm('Are you sure you want to clear the database of all its nodes and links? This action cannot be undone.')) {
|
|
fetch('/api/cleardb', {
|
|
method: 'POST',
|
|
})
|
|
.then(response => response.json())
|
|
.then(data => {
|
|
if (data.message) {
|
|
alert(data.message);
|
|
} else {
|
|
alert('Database cleared successfully.');
|
|
}
|
|
// Set a timeout to refresh the page after 2 seconds
|
|
setTimeout(function() {
|
|
window.location.reload();
|
|
}, 2000);
|
|
})
|
|
.catch((error) => {
|
|
console.error('Error:', error);
|
|
alert('An error occurred while trying to clear the database.');
|
|
});
|
|
}
|
|
});
|
|
|
|
// Add an event listener to the search button
|
|
searchButton.addEventListener('click', function() {
|
|
const url = urlInput.value.trim();
|
|
const maxDepth = depthInput.value;
|
|
|
|
if (url && url.startsWith('http')) {
|
|
// Send the data to /api/search
|
|
sendSearchData(url, maxDepth);
|
|
} else {
|
|
alert('Invalid URL provided. Please enter a valid URL starting with "http".');
|
|
}
|
|
});
|
|
|
|
function sendSearchData(url, maxDepth) {
|
|
fetch('/api/search', {
|
|
method: 'POST',
|
|
headers: {
|
|
'Content-Type': 'application/json'
|
|
},
|
|
body: JSON.stringify({
|
|
starting_url: url,
|
|
max_depth: maxDepth
|
|
})
|
|
})
|
|
.then(response => {
|
|
if (response.ok) {
|
|
return response.json();
|
|
}
|
|
throw new Error('Network response was not ok.');
|
|
})
|
|
.then(data => {
|
|
console.log('Success:', data);
|
|
})
|
|
.catch((error) => {
|
|
console.error('Error:', error);
|
|
});
|
|
}
|
|
|
|
// Add event listener to the toggle button
|
|
const toggleButton = document.getElementById('toggle-simulation');
|
|
let isSimulationPaused = false;
|
|
|
|
toggleButton.addEventListener('click', function() {
|
|
if (isSimulationPaused) {
|
|
// If simulation is paused, resume it
|
|
toggleButton.textContent = 'Pause';
|
|
simulation.alpha(1).restart();
|
|
} else {
|
|
// If simulation is running, pause it
|
|
toggleButton.textContent = 'Play';
|
|
simulation.alpha(0);
|
|
}
|
|
isSimulationPaused = !isSimulationPaused;
|
|
});
|
|
|
|
const scaleSlider = document.getElementById('scale-slider');
|
|
|
|
// Add an event listener to the slider
|
|
scaleSlider.addEventListener('input', function() {
|
|
const scaleValue = parseFloat(scaleSlider.value);
|
|
|
|
// Adjust the link distance scale based on the slider value
|
|
simulation.force("link").distance(d => 100 * scaleValue); // You can adjust the multiplier as needed
|
|
|
|
// Restart the simulation with the updated link distance
|
|
simulation.alpha(1).restart();
|
|
});
|
|
|
|
let transform = d3.zoomIdentity;
|
|
let nodes = [];
|
|
let edges = [];
|
|
|
|
canvas.addEventListener('mousemove', function(event) {
|
|
const rect = canvas.getBoundingClientRect();
|
|
const scaleX = canvas.width / rect.width;
|
|
const scaleY = canvas.height / rect.height;
|
|
|
|
const mouseX = (event.clientX - rect.left) * scaleX;
|
|
const mouseY = (event.clientY - rect.top) * scaleY;
|
|
|
|
let hoverNode = null;
|
|
const hoverRadius = 10;
|
|
|
|
nodes.forEach(node => {
|
|
const dx = mouseX - transform.applyX(node.x);
|
|
const dy = mouseY - transform.applyY(node.y);
|
|
if (dx * dx + dy * dy < hoverRadius * hoverRadius) {
|
|
hoverNode = node;
|
|
}
|
|
});
|
|
|
|
if (hoverNode) {
|
|
updateTooltipPosition(event.clientX, event.clientY);
|
|
showTooltip(hoverNode, event.clientX, event.clientY);
|
|
} else {
|
|
hideTooltip();
|
|
}
|
|
});
|
|
|
|
function updateTooltipPosition(x, y) {
|
|
let tooltip = document.getElementById('tooltip');
|
|
if (tooltip) {
|
|
tooltip.style.left = `${x}px`;
|
|
tooltip.style.top = `${y}px`;
|
|
}
|
|
}
|
|
|
|
function showTooltip(node, x, y) {
|
|
let tooltip = document.getElementById('tooltip');
|
|
if (!tooltip) {
|
|
tooltip = document.createElement('div');
|
|
tooltip.id = 'tooltip';
|
|
document.body.appendChild(tooltip);
|
|
}
|
|
|
|
// Get URLs that connect to the node
|
|
const connectionsFrom = edges
|
|
.filter(edge => edge.target === node)
|
|
.map(edge => edge.source.url);
|
|
|
|
// Get URLs that lead from this node
|
|
const connectionsTo = edges
|
|
.filter(edge => edge.source === node)
|
|
.map(edge => edge.target.url);
|
|
|
|
// Truncate the lists if they are longer than 8
|
|
const maxLinksToShow = 8;
|
|
const additionalFrom = connectionsFrom.length > maxLinksToShow ? `...and ${connectionsFrom.length - maxLinksToShow} more` : '';
|
|
const additionalTo = connectionsTo.length > maxLinksToShow ? `...and ${connectionsTo.length - maxLinksToShow} more` : '';
|
|
connectionsFrom.length = Math.min(connectionsFrom.length, maxLinksToShow);
|
|
connectionsTo.length = Math.min(connectionsTo.length, maxLinksToShow);
|
|
|
|
// Create HTML content for the tooltip
|
|
let tooltipContent = `<strong>Node URL:</strong> ${node.url}<br>`;
|
|
|
|
if (connectionsFrom.length > 0) {
|
|
tooltipContent += `<strong>Connections from:</strong> ${connectionsFrom.join(', ')}${additionalFrom}<br>`;
|
|
}
|
|
|
|
if (connectionsTo.length > 0) {
|
|
tooltipContent += `<strong>Connections To:</strong> ${connectionsTo.join(', ')}${additionalTo}<br>`;
|
|
}
|
|
|
|
tooltip.style.display = 'block';
|
|
tooltip.style.position = 'absolute';
|
|
tooltip.style.left = `${x}px`;
|
|
tooltip.style.top = `${y}px`;
|
|
tooltip.innerHTML = tooltipContent;
|
|
tooltip.style.pointerEvents = 'none';
|
|
tooltip.style.padding = '8px';
|
|
tooltip.style.background = 'rgba(255, 255, 255, 0.75)';
|
|
tooltip.style.border = '1px solid #ddd';
|
|
tooltip.style.borderRadius = '4px';
|
|
}
|
|
|
|
function hideTooltip() {
|
|
const tooltip = document.getElementById('tooltip');
|
|
if (tooltip) {
|
|
tooltip.style.display = 'none';
|
|
}
|
|
}
|
|
|
|
const simulation = d3.forceSimulation()
|
|
.force('charge', d3.forceManyBody().strength(-50)) // Increased repulsive force
|
|
.force("link", d3.forceLink().id(d => d.id).distance(50))
|
|
.force("center", d3.forceCenter(canvas.width / 2, canvas.height / 2))
|
|
.on("tick", draw);
|
|
|
|
|
|
|
|
|
|
const defaultZoomScale = 0.05;
|
|
|
|
// Calculate the center position
|
|
const centerX = canvas.width / 2;
|
|
const centerY = canvas.height / 2;
|
|
|
|
// Calculate the translation needed to center the content
|
|
const translateX = centerX * (1 - defaultZoomScale);
|
|
const translateY = centerY * (1 - defaultZoomScale);
|
|
|
|
// Modify the zoomHandler to start with the default scale and center the content
|
|
const zoomHandler = d3.zoom()
|
|
.on("zoom", (event) => {
|
|
transform = event.transform;
|
|
draw();
|
|
});
|
|
|
|
d3.select(canvas).call(zoomHandler);
|
|
zoomHandler.transform(d3.select(canvas), d3.zoomIdentity.translate(translateX, translateY).scale(defaultZoomScale));
|
|
|
|
function draw() {
|
|
ctx.save();
|
|
ctx.clearRect(0, 0, canvas.width, canvas.height);
|
|
ctx.translate(transform.x, transform.y);
|
|
ctx.scale(transform.k, transform.k);
|
|
|
|
// Calculate viewport boundaries
|
|
const minX = -transform.x / transform.k;
|
|
const minY = -transform.y / transform.k;
|
|
const maxX = minX + canvas.width / transform.k;
|
|
const maxY = minY + canvas.height / transform.k;
|
|
|
|
// Filter and render only visible nodes
|
|
const visibleNodes = nodes.filter(node => isNodeVisible(node, minX, minY, maxX, maxY));
|
|
|
|
// Create a set of visible node IDs for efficient lookup
|
|
const visibleNodeIds = new Set(visibleNodes.map(node => node.id));
|
|
|
|
// Filter and render only visible edges that connect visible nodes
|
|
const visibleEdges = edges.filter(edge =>
|
|
visibleNodeIds.has(edge.source.id) && visibleNodeIds.has(edge.target.id)
|
|
);
|
|
|
|
visibleEdges.forEach(drawLink);
|
|
visibleNodes.forEach(drawNode);
|
|
|
|
ctx.restore();
|
|
}
|
|
|
|
function isNodeVisible(node, minX, minY, maxX, maxY) {
|
|
return (
|
|
node.x >= minX &&
|
|
node.x <= maxX &&
|
|
node.y >= minY &&
|
|
node.y <= maxY
|
|
);
|
|
}
|
|
|
|
function isEdgeVisible(edge, minX, minY, maxX, maxY) {
|
|
// Check if either of the edge's endpoints is inside the viewport
|
|
return (
|
|
isNodeVisible(edge.source, minX, minY, maxX, maxY) ||
|
|
isNodeVisible(edge.target, minX, minY, maxX, maxY)
|
|
);
|
|
}
|
|
|
|
function domainToColor(url) {
|
|
const domainColors = {
|
|
'twitter': '#1DA1F2',
|
|
'facebook': '#3b5998',
|
|
'youtube': '#DB4437',
|
|
'github': '#013220',
|
|
'google': '#f0f0f0',
|
|
'instagram': '#E1306C',
|
|
'linkedin': '#0077B5',
|
|
'whatsapp': '#25D366',
|
|
'spotify': '#1DB954',
|
|
'medium': '#12100E',
|
|
'pinterest': '#BD081C',
|
|
'reddit': '#FF4500',
|
|
'twitch': '#6441A4',
|
|
'steam': ' #66c0f4',
|
|
|
|
// File Types
|
|
'.txt': '#a9a9a9', // Gray for Text files
|
|
'Image Files': '#FFA500', // Orange for Image files
|
|
'Audio Files': '#1DB954', // Green for Audio files
|
|
'Video Files': '#FF3333', // Red for Video files
|
|
'Compressed/Executable Files': '#FF0000' // Bright red for certain file types
|
|
};
|
|
|
|
const imageExtensions = ['.png', '.jpg', '.jpeg', '.gif', '.bmp', '.tiff', '.svg'];
|
|
const audioExtensions = ['.mp3', '.wav', '.aac'];
|
|
const videoExtensions = ['.mp4', '.avi', '.mov'];
|
|
const compressedExecutableExtensions = ['.rar', '.zip', '.7z', '.exe'];
|
|
|
|
for (const extension of imageExtensions) {
|
|
if (url.endsWith(extension)) {
|
|
return domainColors['Image Files'];
|
|
}
|
|
}
|
|
|
|
for (const extension of audioExtensions) {
|
|
if (url.endsWith(extension)) {
|
|
return domainColors['Audio Files'];
|
|
}
|
|
}
|
|
|
|
for (const extension of videoExtensions) {
|
|
if (url.endsWith(extension)) {
|
|
return domainColors['Video Files'];
|
|
}
|
|
}
|
|
|
|
for (const extension of compressedExecutableExtensions) {
|
|
if (url.endsWith(extension)) {
|
|
return domainColors['Compressed/Executable Files'];
|
|
}
|
|
}
|
|
|
|
for (const domain in domainColors) {
|
|
if (url.includes(domain)) {
|
|
return domainColors[domain];
|
|
}
|
|
}
|
|
|
|
return '#aaa'; // Default color if no match is found
|
|
}
|
|
|
|
function updateLegend() {
|
|
const domainColors = {
|
|
'twitter/x': '#1DA1F2',
|
|
'facebook': '#3b5998',
|
|
'youtube': '#DB4437',
|
|
'github': '#013220',
|
|
'google': '#f0f0f0',
|
|
'instagram': '#E1306C',
|
|
'linkedin': '#0077B5',
|
|
'whatsapp': '#25D366',
|
|
'spotify': '#1DB954',
|
|
'medium': '#12100E',
|
|
'pinterest': '#BD081C',
|
|
'reddit': '#FF4500',
|
|
'twitch': '#6441A4',
|
|
'steam': ' #66c0f4',
|
|
|
|
|
|
'.txt Files': '#a9a9a9', // Gray for Text files
|
|
'All Audio Files': '#1DB954', // Green for Audio files
|
|
'All Video Files': '#FF3333', // Red for Video files
|
|
'All Image Files': '#FFA500', // Orange for Image files
|
|
'Compressed/Executable Files': '#FF0000' // Bright red for certain file types
|
|
};
|
|
|
|
|
|
const legend = document.getElementById('legend');
|
|
legend.innerHTML = ''; // Clear existing legend items
|
|
|
|
Object.keys(domainColors).forEach(domain => {
|
|
const colorBox = document.createElement('div');
|
|
colorBox.className = 'legend-color-box';
|
|
colorBox.style.background = domainColors[domain];
|
|
|
|
const listItem = document.createElement('li');
|
|
listItem.appendChild(colorBox);
|
|
listItem.appendChild(document.createTextNode(domain));
|
|
|
|
legend.appendChild(listItem);
|
|
});
|
|
}
|
|
updateLegend();
|
|
function drawNode(d) {
|
|
let color = domainToColor(d.url);
|
|
let radius = 5;
|
|
|
|
if (d.isOriginalSearch) {
|
|
color = 'gold';
|
|
radius = 10;
|
|
}
|
|
|
|
ctx.beginPath();
|
|
ctx.arc(d.x, d.y, radius, 0, 2 * Math.PI);
|
|
ctx.fillStyle = color;
|
|
ctx.fill();
|
|
|
|
// Add a white ring around the node
|
|
ctx.beginPath();
|
|
ctx.arc(d.x, d.y, radius + 2, 0, 2 * Math.PI);
|
|
ctx.strokeStyle = 'white'; // Set the ring color to white
|
|
ctx.lineWidth = 0.5; // Set the ring width
|
|
ctx.stroke();
|
|
}
|
|
|
|
function drawLink(l) {
|
|
const sourceColor = domainToColor(l.source.url);
|
|
const targetColor = domainToColor(l.target.url);
|
|
|
|
// Set the opacity for the link
|
|
ctx.globalAlpha = 0.2; // You can adjust the opacity as needed
|
|
ctx.beginPath();
|
|
ctx.moveTo(l.source.x, l.source.y);
|
|
ctx.lineTo(l.target.x, l.target.y);
|
|
ctx.strokeStyle = sourceColor === targetColor ? sourceColor : "#aaa";
|
|
ctx.stroke();
|
|
|
|
// Reset the globalAlpha to 1.0 after drawing the link
|
|
ctx.globalAlpha = 1.0;
|
|
}
|
|
|
|
async function fetchData() {
|
|
const nodesResponse = await fetch('/api/nodes');
|
|
const edgesResponse = await fetch('/api/edges');
|
|
nodes = await nodesResponse.json();
|
|
edges = await edgesResponse.json();
|
|
|
|
// Fetch data from the original_searches table
|
|
const originalSearchResponse = await fetch('/api/original_search_ids');
|
|
const originalSearchData = await originalSearchResponse.json();
|
|
|
|
// Fetch URLs from original_searches and create a Set for faster lookup
|
|
const originalSearchUrls = new Set();
|
|
originalSearchData.forEach(entry => {
|
|
originalSearchUrls.add(entry.url);
|
|
});
|
|
|
|
nodes.forEach(node => {
|
|
if (originalSearchUrls.has(node.url)) {
|
|
node.isOriginalSearch = true;
|
|
}
|
|
});
|
|
|
|
simulation
|
|
.nodes(nodes)
|
|
.force("link")
|
|
.links(edges);
|
|
|
|
draw();
|
|
}
|
|
|
|
canvas.addEventListener('contextmenu', function(event) {
|
|
event.preventDefault();
|
|
|
|
const rect = canvas.getBoundingClientRect();
|
|
const scaleX = canvas.width / rect.width;
|
|
const scaleY = canvas.height / rect.height;
|
|
|
|
const mouseX = (event.clientX - rect.left) * scaleX;
|
|
const mouseY = (event.clientY - rect.top) * scaleY;
|
|
|
|
nodes.forEach(node => {
|
|
const dx = mouseX - transform.applyX(node.x);
|
|
const dy = mouseY - transform.applyY(node.y);
|
|
if (dx * dx + dy * dy < (5 * transform.k) ** 2) {
|
|
openInNewTab(node.url);
|
|
}
|
|
});
|
|
});
|
|
|
|
function openInNewTab(url) {
|
|
window.open(url, '_blank').focus();
|
|
}
|
|
|
|
canvas.addEventListener('click', function(event) {
|
|
const rect = canvas.getBoundingClientRect();
|
|
const scaleX = canvas.width / rect.width;
|
|
const scaleY = canvas.height / rect.height;
|
|
|
|
const mouseX = (event.clientX - rect.left) * scaleX;
|
|
const mouseY = (event.clientY - rect.top) * scaleY;
|
|
|
|
nodes.forEach(node => {
|
|
const dx = mouseX - transform.applyX(node.x);
|
|
const dy = mouseY - transform.applyY(node.y);
|
|
if (dx * dx + dy * dy < (5 * transform.k) ** 2) {
|
|
confirmAndSendNodeUrl(node.url);
|
|
}
|
|
});
|
|
});
|
|
|
|
function confirmAndSendNodeUrl(url) {
|
|
const confirmation = confirm(`This will start a search from ${url}, are you sure?`);
|
|
if (confirmation) {
|
|
sendNodeUrl(url);
|
|
}
|
|
}
|
|
|
|
function sendNodeUrl(url) {
|
|
const maxDepth = 4;
|
|
|
|
fetch('/api/search', {
|
|
method: 'POST',
|
|
headers: {
|
|
'Content-Type': 'application/json'
|
|
},
|
|
body: JSON.stringify({
|
|
starting_url: url,
|
|
max_depth: maxDepth
|
|
})
|
|
})
|
|
.then(response => {
|
|
if (response.ok) {
|
|
return response.json();
|
|
}
|
|
throw new Error('Network response was not ok.');
|
|
})
|
|
.then(data => {
|
|
console.log('Success:', data);
|
|
})
|
|
.catch((error) => {
|
|
console.error('Error:', error);
|
|
});
|
|
}
|
|
|
|
fetchData();
|
|
|
|
window.addEventListener('resize', resizeCanvas);
|
|
|
|
function resizeCanvas() {
|
|
canvas.width = window.innerWidth;
|
|
canvas.height = window.innerHeight;
|
|
|
|
// Additional code to reposition or redraw elements if necessary
|
|
simulation.force("center", d3.forceCenter(canvas.width / 2, canvas.height / 2));
|
|
draw();
|
|
}
|
|
|
|
// Call resizeCanvas initially to set up canvas size
|
|
resizeCanvas();
|
|
</script>
|
|
</body>
|
|
</html>
|