suuuper alpha. might fall over. v0.0.1.

binhop.py

@@ -6,32 +6,45 @@ import time
 import tempfile
 import binwalk
 import os
-
+from aiohttp_compress import compress_middleware
-
 from aiohttp import web
 
+
 async def scan_file(filename, base_dir):
     try:
-        scan = binwalk.scan(filename, signature=True, quiet=True, extract=True, matryoshka=True, remove_after_execute=False, directory=base_dir)
+        scan = binwalk.scan(
+            filename,
+            signature=True,
+            quiet=True,
+            extract=True,
+            matryoshka=True,
+            remove_after_execute=False,
+            directory=base_dir,
+        )
         return scan
     except binwalk.ModuleException as e:
         print("Critical failure: ", e)
 
+
 async def build_listing(path):
     result = {}
+    num_files, num_dirs = 0, 0
     for item in os.listdir(path):
         item_path = os.path.join(path, item)
         if os.path.isdir(item_path):
-            result[item] = await build_listing(item_path)
+            files, dirs, result[item] = await build_listing(item_path)
+            num_files += files
+            num_dirs += 1 + dirs
         else:
             result[item] = {"s": os.path.getsize(item_path)}
-            print(item)
+            num_files += 1
-    return result
+    return num_files, num_dirs, result
+
 
 async def upload_file(request):
     reader = await request.multipart()
     field = await reader.next()
-    assert field.name == 'file'
+    assert field.name == "file"
 
     start_time = time.time()
     filename = field.filename
@@ -39,8 +52,7 @@ async def upload_file(request):
     sha1_hash = hashlib.sha1()
     md5_hash = hashlib.md5()
 
-
+    temp_file = tempfile.NamedTemporaryFile(mode="ab", delete=False)
-    temp_file = tempfile.NamedTemporaryFile(mode='ab', delete=False)
 
     while True:
         chunk = await field.read_chunk()
@@ -59,72 +71,88 @@ async def upload_file(request):
     finally:
         os.unlink(temp_file.name)
 
-    carved = []
+    carved, summary = [], []
     for sig in sigs.results:
         tmp_path = sig.file.path
         if tmp_path in extractor:
             if sig.offset in extractor[tmp_path].carved:
                 end_offset = sig.offset + os.path.getsize(extractor[tmp_path].carved[sig.offset])
-                print("Carved data from offsets 0x%X-0x%X to %s" % (sig.offset, end_offset, extractor[tmp_path].carved[sig.offset]))
+                summary.append(
+                    "Carved data from offsets 0x%X-0x%X to %s"
+                    % (sig.offset, end_offset, extractor[tmp_path].carved[sig.offset])
+                )
                 carved.append({"start": sig.offset, "end": end_offset, "d": sig.description})
             if sig.offset in extractor[tmp_path].extracted:
                 extracted_files = [x for x in extractor[tmp_path].extracted[sig.offset].files if os.path.isfile(x)]
                 extracted_dirs = [x for x in extractor[tmp_path].extracted[sig.offset].files if os.path.isdir(x)]
-                print("Extracted %d files and %d directories from offset 0x%X to '%s' using '%s'" % (len(extracted_files), len(extracted_dirs), sig.offset, extractor[tmp_path].extracted[sig.offset].files[0], sigs.extractor.output[tmp_path].extracted[sig.offset].command))
+                summary.append(
-                for i in extractor[tmp_path].extracted[sig.offset].files:
+                    "Extracted %d files and %d directories from offset 0x%X to '%s' using '%s'"
-                    print(f"  File: {i}")
+                    % (
-    # listing = await build_listing(working_dir.name)
+                        len(extracted_files),
-    # print(listing)
+                        len(extracted_dirs),
+                        sig.offset,
+                        extractor[tmp_path].extracted[sig.offset].files[0],
+                        sigs.extractor.output[tmp_path].extracted[sig.offset].command,
+                    )
+                )
+    num_files, num_dirs, listing = await build_listing(working_dir.name)
     working_dir.cleanup()
 
     response_data = {
-        'meta': {
+        "meta": {
-            'name': filename,
+            "name": filename,
-            'sizeb': file_size,
+            "sizeb": file_size,
-            'sha1': sha1_hash.hexdigest(),
+            "sha1": sha1_hash.hexdigest(),
-            'md5': md5_hash.hexdigest(),
+            "md5": md5_hash.hexdigest(),
-            'sig_quant': len(sigs.magic.signatures)
+            "sig_quant": len(sigs.magic.signatures),
+            "files": num_files,
+            "dirs": num_dirs,
         },
-        'offsets': carved
+        "offsets": carved,
+        "ls": listing,
+        "ql": summary,
     }
 
     processing_time = time.time() - start_time
     minutes, seconds = divmod(processing_time, 60)
     milliseconds = processing_time - int(processing_time)
-    response_data['meta']['duration'] = f"{int(minutes):02d}:{int(seconds):02d}.{int(milliseconds * 1000):03d}"
+    response_data["meta"]["duration"] = f"{int(minutes):02d}:{int(seconds):02d}.{int(milliseconds * 1000):03d}"
     return web.json_response(response_data)
 
 
 async def serve_index(request):
-    return web.FileResponse('index.html')
+    return web.FileResponse("index.html")
 
 
 async def serve_static(request):
     path = request.path[1:]
-    if not path.startswith('static/'):
+    if not path.startswith("static/"):
         return web.HTTPNotFound()
     return web.FileResponse(path)
 
 
 async def main():
     app = web.Application()
+    app.middlewares.append(compress_middleware)
 
-    app.add_routes([
+    app.add_routes(
-        web.get('/', serve_index),
+        [
-        web.post('/api/upload_file', upload_file),
+            web.get("/", serve_index),
-        web.get('/static/{tail:.*}', serve_static)
+            web.post("/api/upload_file", upload_file),
-    ])
+            web.get("/static/{tail:.*}", serve_static),
+        ]
+    )
 
     runner = web.AppRunner(app)
     await runner.setup()
-    site = web.TCPSite(runner, 'localhost', 8080)
+    site = web.TCPSite(runner, "localhost", 8080)
     await site.start()
 
-    print('binhop is running at http://localhost:8080')
+    print("binhop is running at http://localhost:8080")
     await asyncio.Event().wait()
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     loop = asyncio.new_event_loop()
     asyncio.set_event_loop(loop)
     asyncio.run(main())

index.html

@@ -4,44 +4,8 @@
     <meta charset="UTF-8">
     <title>binhop</title>
     <link rel="icon" type="image/ico" href="static/favicon.ico">
+    <link rel="stylesheet" href="static/binhop.css">
     <link rel="stylesheet" href="static/pico.min.css">
-    <style>
-        body {
-          color: #fff;
-        }
-        .header {
-          max-width: 600px;
-          margin: 0 auto;
-          text-align: center;
-        }
-        .container {
-          max-width: 1200px;
-          margin: 0 auto;
-        }
-        .logo {
-          margin-top: 50px;
-          margin-bottom: 30px;
-          width: 200px;
-        }
-        .button-center {
-          max-width: 600px;
-          margin: 0 auto;
-          text-align: center;
-        }
-        button {
-          background-color: #3498db;
-          color: #fff;
-          border: none;
-          padding: 10px 20px;
-          font-size: 16px;
-        }
-        button:hover {
-          background-color: #2980b9;
-        }
-        pre {
-          white-space: pre-wrap;
-        }
-      </style>
   </head>
   <body>
     <a href="https://github.com/DarrylNixon/binhop" class="github-corner" aria-label="View source on GitHub"><svg width="80" height="80" viewBox="0 0 250 250" style="fill:#fff; color:#11191f; position: absolute; top: 0; border: 0; right: 0;" aria-hidden="true"><path d="M0,0 L115,115 L130,115 L142,142 L250,250 L250,0 Z"></path><path d="M128.3,109.0 C113.8,99.7 119.0,89.6 119.0,89.6 C122.0,82.7 120.5,78.6 120.5,78.6 C119.2,72.0 123.4,76.3 123.4,76.3 C127.3,80.9 125.5,87.3 125.5,87.3 C122.9,97.6 130.6,101.9 134.4,103.2" fill="currentColor" style="transform-origin: 130px 106px;" class="octo-arm"></path><path d="M115.0,115.0 C114.9,115.1 118.7,116.5 119.8,115.4 L133.7,101.6 C136.9,99.2 139.9,98.4 142.2,98.6 C133.8,88.0 127.5,74.4 143.8,58.0 C148.5,53.4 154.0,51.2 159.7,51.0 C160.3,49.4 163.2,43.6 171.4,40.1 C171.4,40.1 176.1,42.5 178.8,56.2 C183.1,58.6 187.2,61.8 190.9,65.4 C194.5,69.0 197.7,73.2 200.1,77.6 C213.8,80.2 216.3,84.9 216.3,84.9 C212.7,93.1 206.9,96.0 205.4,96.6 C205.1,102.4 203.0,107.8 198.3,112.5 C181.9,128.9 168.3,122.5 157.7,114.1 C157.9,116.9 156.7,120.9 152.7,124.9 L141.0,136.5 C139.8,137.7 141.6,141.9 141.8,141.8 Z" fill="currentColor" class="octo-body"></path></svg></a><style>.github-corner:hover .octo-arm{animation:octocat-wave 560ms ease-in-out}@keyframes octocat-wave{0%,100%{transform:rotate(0)}20%,60%{transform:rotate(-25deg)}40%,80%{transform:rotate(10deg)}}@media (max-width:500px){.github-corner:hover .octo-arm{animation:none}.github-corner .octo-arm{animation:octocat-wave 560ms ease-in-out}}</style>
@@ -57,6 +21,8 @@
             <button id="upload-button" onclick="return upload_file()">give me the blob</button>
         </div>
     </div>
+    <div id="footer">v0.0.1</div>
     <script src="static/binhop.js"></script>
+    <script src="static/chroma.min.js"></script>
   </body>
 </html>

requirements.txt

@@ -1,2 +1,3 @@
 aiohttp
+aiohttp-compress
 binwalk

static/binhop.css

@@ -0,0 +1,109 @@
+body {
+    color: #fff;
+}
+.header {
+    max-width: 600px;
+    margin: 0 auto;
+    text-align: center;
+}
+.container {
+    max-width: 1200px;
+    margin: 0 auto;
+}
+.logo {
+    margin-top: 50px;
+    margin-bottom: 30px;
+    width: 200px;
+}
+.button-center {
+    max-width: 600px;
+    margin: 0 auto;
+    text-align: center;
+}
+button {
+    background-color: #3498db;
+    color: #fff;
+    border: none;
+    padding: 10px 20px;
+    font-size: 16px;
+}
+button:hover {
+    background-color: #2980b9;
+}
+pre {
+    white-space: pre-wrap;
+}
+#footer {
+    position: fixed;
+    bottom: 10px;
+    right: 10px;
+    background-color:#263745;
+    padding: .2em;
+    opacity: 0.4;
+}
+/* Yet unused, from https://iamkate.com/code/tree-views/ */
+/* .tree{
+    --spacing : 1.5rem;
+    --radius : 10px;
+}
+.tree li{
+    display : block;
+    position : relative;
+    padding-left : calc(2 * var(--spacing) - var(--radius) - 2px);
+}
+.tree ul{
+    margin-left : calc(var(--radius) - var(--spacing));
+    padding-left : 0;
+}
+.tree ul li{
+    border-left : 2px solid #ddd;
+}
+.tree ul li:last-child{
+    border-color : transparent;
+}
+.tree ul li::before{
+    content : '';
+    display : block;
+    position : absolute;
+    top : calc(var(--spacing) / -2);
+    left : -2px;
+    width : calc(var(--spacing) + 2px);
+    height : calc(var(--spacing) + 1px);
+    border : solid #ddd;
+    border-width : 0 0 2px 2px;
+}
+.tree summary{
+    display : block;
+    cursor : pointer;
+}
+.tree summary::marker, .tree summary::-webkit-details-marker{
+    display : none;
+}
+.tree summary:focus{
+    outline : none;
+}
+.tree summary:focus-visible{
+    outline : 1px dotted #000;
+}
+.tree li::after, .tree summary::before{
+    content : '';
+    display : block;
+    position : absolute;
+    top : calc(var(--spacing) / 2 - var(--radius));
+    left : calc(var(--spacing) - var(--radius) - 1px);
+    width : calc(2 * var(--radius));
+    height : calc(2 * var(--radius));
+    border-radius : 50%;
+    background : #ddd;
+}
+.tree summary::before{
+    content : '+';
+    z-index : 1;
+    background : #696;
+    color : #fff;
+    line-height : calc(2 * var(--radius) - 2px);
+    text-align : center;
+}
+.tree details[open] > summary::before{
+    content : '−';
+} */

static/binhop.js

@@ -1,40 +1,197 @@
 function draw_bytes(data) {
     const canvas = document.getElementById("blob");
     const ctx = canvas.getContext("2d");
+    const colorScale = chroma.scale(['#4c6c81', '#9c2f2f']).mode('lch');
 
-    // canvas.width = canvas.parentNode.width;
+    const numBlocks = Math.ceil(data.meta.sizeb / 1200);
-    // canvas.height = canvas.parentNode.height;
+    const blockSize = canvas.width / numBlocks;
-
-    const blockSize = 2;
-    const blockPadding = 1;
 
     ctx.fillStyle = "#ddd";
+    ctx.fillRect(0, 0, canvas.width, canvas.height);
 
-    const numBlocks = Math.ceil(data.meta.sizeb / (blockSize + blockPadding));
+    // console.log("size: " + data.meta.sizeb);
-    for (let i = 0; i < numBlocks; i++) {
+    // console.log("numblocks: " + numBlocks);
-        const x = i * (blockSize + blockPadding);
+    data.offsets.forEach((offset, idx) => {
-        ctx.fillRect(x, 0, blockSize, canvas.height);
+        const start = Math.floor((offset.start / data.meta.sizeb) * numBlocks);
-    }
+        const end = Math.ceil((offset.end / data.meta.sizeb) * numBlocks);
+        // console.log(offset.d + ": " + start + ", " + end + " | " + offset.start + ", " + offset.end);
+        ctx.fillStyle = colorScale(idx / data.offsets.length).alpha(0.2);
+        ctx.fillRect(start * blockSize, 0, (end - start) * blockSize, canvas.height);
+    });
 
-    ctx.fillStyle = "blue";
+    canvas.addEventListener('mousemove', function(event) {
-    data.offsets.forEach((offset) => {
+        const rect = canvas.getBoundingClientRect();
-        const start = Math.floor(offset.start / (blockSize + blockPadding));
+        const mouseX = event.clientX - rect.left;
-        const end = Math.ceil(offset.end / (blockSize + blockPadding));
+        const mouseY = event.clientY - rect.top;
-        for (let i = start; i < end; i++) {
+    
-            const x = i * (blockSize + blockPadding);
+        const tooltips = [];
-            ctx.fillRect(x, 0, blockSize, canvas.height);
+    
-        }
+        data.offsets.forEach((offset, idx) => {
+            const start = Math.floor((offset.start / data.meta.sizeb) * numBlocks);
+            const end = Math.ceil((offset.end / data.meta.sizeb) * numBlocks);
+            const rectX = start * blockSize;
+            const rectY = 0;
+            const rectWidth = (end - start) * blockSize;
+            const rectHeight = canvas.height;
+    
+            if (mouseX >= rectX && mouseX <= rectX + rectWidth && mouseY >= rectY && mouseY <= rectY + rectHeight) {
+                const tooltipText = `     ${first_substr(offset.d)}`;
+                const tooltipWidth = ctx.measureText(tooltipText).width + 20;
+                const tooltipHeight = 30;
+    
+                let tooltipX = mouseX + 10;
+                let tooltipY = mouseY - tooltipHeight - 10;
+    
+                if (tooltipX + tooltipWidth > canvas.width) {
+                    tooltipX = canvas.width - tooltipWidth;
+                }
+    
+                if (tooltipY < 0) {
+                    tooltipY = 0;
+                }
+    
+                // Check for overlapping tooltips
+                for (let i = 0; i < tooltips.length; i++) {
+                    const otherTooltip = tooltips[i];
+    
+                    if (tooltipX < otherTooltip.x + otherTooltip.width &&
+                        tooltipX + tooltipWidth > otherTooltip.x &&
+                        tooltipY < otherTooltip.y + otherTooltip.height &&
+                        tooltipY + tooltipHeight > otherTooltip.y) {
+    
+                        // Overlapping, adjust the position
+                        tooltipX = otherTooltip.x + otherTooltip.width + 10;
+    
+                        if (tooltipX + tooltipWidth > canvas.width) {
+                            tooltipX = mouseX - tooltipWidth - 10;
+                        }
+    
+                        if (tooltipX < 0) {
+                            tooltipX = 0;
+                        }
+    
+                        tooltipY = otherTooltip.y;
+                    }
+                }
+    
+                tooltips.push({
+                    x: tooltipX,
+                    y: tooltipY,
+                    width: tooltipWidth,
+                    height: tooltipHeight
+                });
+    
+                ctx.fillStyle = '#fff';
+                ctx.fillRect(tooltipX, tooltipY, tooltipWidth, tooltipHeight);
+    
+                ctx.fillStyle = '#000';
+                ctx.font = '.6em Arial';
+                ctx.fillText(tooltipText, tooltipX + 10, tooltipY + 20);
+                // colored circle
+                const circleSize = 10;
+                ctx.fillStyle = colorScale(idx / data.offsets.length).alpha(0.2);
+                ctx.beginPath();
+                ctx.arc(tooltipX + 15, tooltipY + 15, circleSize / 2, 0, 2 * Math.PI);
+                ctx.fill();
+                // re-draw the rest
+                ctx.fillStyle = colorScale(idx / data.offsets.length).alpha(0.2);
+                ctx.fillRect(start * blockSize, 0, (end - start) * blockSize, canvas.height);
+            }
+        });
+    });
+    
+    canvas.addEventListener('mouseout', function(event) {
+        ctx.clearRect(0, 0, canvas.width, canvas.height);
+        data.offsets.forEach((offset, idx) => {
+            const start = Math.floor((offset.start / data.meta.sizeb) * numBlocks);
+            const end = Math.ceil((offset.end / data.meta.sizeb) * numBlocks);
+            ctx.fillStyle = colorScale(idx / data.offsets.length).alpha(0.2);
+            ctx.fillRect(start * blockSize, 0, (end - start) * blockSize, canvas.height);
+        });
     });
 }
 
+// function create_listing(listing) {
+//     const ul = document.createElement('ul');
+//     ul.classList.add("tree-padding", "tree-vertical-lines", "tree-horizontal-lines", "tree-summaries", "tree-markers", "tree-buttons")
+
+//     for (const key in listing) {
+//         const value = listing[key];
+
+//         if (typeof value == 'object') {
+//             const li = document.createElement('li');
+//             const summary = document.createElement('summary');
+//             summary.innerText = key;
+//             li.appendChild(summary);
+
+//             const nestedUl = create_listing(value);
+//             li.appendChild(nestedUl);
+//             ul.appendChild(li);
+//         } else {
+//             const li = document.createElement('li');
+//             const summary = document.createElement('summary');
+//             summary.classList.add('no-pico');
+//             summary.innerText = `${key} (${value.s})`;
+//             li.appendChild(summary);
+//             ul.appendChild(li);
+//         }
+//     }
+//     return ul;
+// }
+
+function file_summaries(data) {
+    const ul = document.createElement('ul');
+    for (const [key, val] of Object.entries(data.ql)) {
+        const li = document.createElement('li');
+        li.textContent = `${val}`;
+        ul.appendChild(li);
+    }
+    return ul;
+}
+
+function create_listing(listing) {
+    const ul = document.createElement('ul');
+
+    for (const key in listing) {
+        const value = listing[key];
+
+        if ('s' in value) {
+            const li = document.createElement('li');
+            li.innerText = `${key} (${bytes_to_human(value.s)})`;
+            ul.appendChild(li);
+        }
+        else { // (typeof value === 'object') {
+            const li = document.createElement('li');
+
+            li.innerText = `${key}/`;
+
+            const nestedUl = create_listing(value);
+
+            li.appendChild(nestedUl);
+            ul.appendChild(li);
+        }
+    }
+    return ul;
+}
+
 function bytes_to_human(bytes) {
+    num = parseInt(bytes);
 	const units = ["B", "KB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB"];
 	let i =0;
-	while (bytes >= 1024 && i < units.length - 1) {
+	while (num >= 1024 && i < units.length - 1) {
-		bytes /= 1024;
+		num /= 1024;
 		i++;
 	}
-	return bytes.toFixed(2) + " " + units[i];
+	return num.toFixed(2) + " " + units[i];
+}
+
+function first_substr(description) {
+    const index = description.indexOf(',');
+    if (index === -1) {
+        return description;
+    } else {
+        return description.substring(0, index);
+    }
 }
 
 async function process_upload(file) {
@@ -52,24 +209,26 @@ async function process_upload(file) {
     });
   
     if (response.ok) {
+    // if (1 === 1) {
       const data = await response.json();
+    //   const json = '{"meta":{"name":"miwifi_r4av2_firmware_6bdd4_2.30.500.bin","sizeb":13632488,"sha1":"6cefd9d5de3b80799d0341b19043d108624dcaca","md5":"639616a5b70983903ccdd019a7a6bdd4","sig_quant":411,"duration":"00:05.457"},"offsets":[{"start":960,"end":13630770,"d":"LZMA compressed data"},{"start":1704660,"end":13621866,"d":"Squashfs filesystem"},{"start":4534008,"end":5267984,"d":"xz compressed data"},{"start":5258380,"end":5267984,"d":"ASCII cpio archive"}]}';
+    //   const data = JSON.parse(json);
+
       const container = document.querySelector('.container');
   
       // visualization
       container.innerHTML = `
-<div class="grid">
+<div id="blob-row">
-<div id="blob-column">
+<canvas id="blob" width="1200px" height="60px" style="background-color: #18232c;"></canvas>
-<canvas id="blob" width="600px" height="600px" style="background-color: #18232c;"></canvas>
 </div>
-<div id="info-column">
+<div id="info-row">
-<small>
+<h4 style="margin-bottom:0.5em;">${data.meta.name}</h4>
-<h1>${data.meta.name}</h1>
 <details open>
     <summary>blob meta</summary>
     <figure>
         <table role="grid">
             <tr>
-                <th><strong>filename:</strong></th>
+                <th><strong>input filename:</strong></th>
                 <td>${data.meta.name}</td>
             </tr>
             <tr>
@@ -77,35 +236,59 @@ async function process_upload(file) {
                 <td>${data.meta.duration}</td>
             </tr>
             <tr>
-                <th><strong>size:</strong></th>
+                <th><strong>file size in bytes:</strong></th>
-                <td>${data.meta["sizeb"]} (${bytes_to_human(data.meta.sizeb)})</td>
+                <td>${data.meta.sizeb} (${bytes_to_human(data.meta.sizeb)})</td>
             </tr>
             <tr>
-                <th><strong>sha1:</strong></th>
+                <th><strong>files discovered:</strong></th>
+                <td>${data.meta.files}</td>
+            </tr>
+            <tr>
+                <th><strong>directories unpacked:</strong></th>
+                <td>${data.meta.dirs}</td>
+            </tr>
+            <tr>
+                <th><strong>signatures tried:</strong></th>
+                <td>${data.meta.sig_quant}</td>
+            </tr>
+            <tr>
+                <th><strong>sha1 hash:</strong></th>
                 <td>${data.meta.sha1}</td>
             </tr>
             <tr>
-                <th><strong>md5:</strong></th>
+                <th><strong>md5 hash:</strong></th>
                 <td>${data.meta.md5}</td>
             </tr>
         </table>
     </figure>
 </details>
-<article>
+
-    <header>Header</header>
+<details open>
-    Body
+    <summary>binwalk extraction log</summary>
-    <footer>Footer</footer>
+    <small>
-</article>
+    <div id="quick-look"></div>
-<article>
+    </small>
-    <header>Header</header>
+</details>
-    Body
+
-    <footer>Footer</footer>
+<details>
-</article>
+    <summary>full file listing</summary>
-<pre>${JSON.stringify(data, null, 2)}</pre>
+    <small>
-</small>
+    <div id="file-list"></div>
-</div>
+    </small>
+</details>
+
+<details>
+    <summary>raw json response (dev)</summary>
+    <pre>${JSON.stringify(data, null, 2)}</pre>
+</details>
 </div>`;
     draw_bytes(data);
+
+    const file_list = document.getElementById('file-list');
+    file_list.appendChild(create_listing(data.ls[Object.keys(data.ls)[0]]));
+
+    const quick_look = document.getElementById('quick-look');
+    quick_look.appendChild(file_summaries(data));
     } else {
       console.error('error uploading file');
     }