logo

deblob

remove binary executables from a directory git clone https://anongit.hacktivis.me/git/deblob.git/

main.ha (14667B)

    

  1. // Copyright © 2019 deblob Authors <https://hacktivis.me/projects/deblob>
  2. // SPDX-License-Identifier: BSD-3-Clause
  3. use bytes;
  4. use encoding::json;
  5. use endian;
  6. use errors;
  7. use fmt;
  8. use fnmatch;
  9. use fs;
  10. use getopt;
  11. use io;
  12. use os;
  13. use path;
  14. use strings;
  16. let excludes: []str = [];
  17. let noop: bool = false;
  18. let check: bool = false;
  19. let json: bool = false;
  21. const beam: []u8 = ['F', 'O', 'R', '1']; // Erlang BEAM
  22. const magic: [_](str, []u8) = [
  23. 	("ELF", [0x7F, 'E', 'L', 'F']),
  24. 	("Unix ar(1)", ['!', '<', 'a', 'r', 'c', 'h', '>', '\n']),
  25. 	("PC-BIOS", [0x55, 0xAA]),
  26. 	("Erlang FOR1 BEAM", ['F', 'O', 'R', '1']),
  27. 	("Java .class / MachO exec", [0xCA, 0xFE, 0xBA, 0xBE]),
  28. 	("WinNT EXE", ['M', 'Z', 0x90, 0x00, 0x03, 0x00, 0x00, 0x00]), // Partial MSDOS stub header (ECMA-335 Common Language Infrastructure)
  29. 	("Lua bytecode", [0x1B, 'L', 'u', 'a']),
  30. 	("Wasm", [0x00, 'a', 's', 'm']),
  31. 	("Apple PEF", ['J', 'o', 'y', '!', 'p', 'e', 'f', 'f']), // Apple Preferred Executable Format
  32. 	("DTB", [0xD0, 0x0D, 0xFE, 0xED]), // Device Tree Blob (OpenFirmware, u-boot, …)
  34. 	// Python *.pyc bytecode magic numbers (defined in importlib/_bootstrap_external.py)
  35. 	("Python pyc 2.7", [0x03, 0xF3, '\r', '\n']), // (62211i little-endian)
  36. 	("Python pyc 3.8", [0x55, 0x0D, '\r', '\n']), // (3413i litte-endian)
  37. 	("Python pyc 3.9", [0x61, 0x0D, '\r', '\n']), // (3425i litte-endian)
  38. 	("Python pyc 3.10", [0x6F, 0x0D, '\r', '\n']), // (3439i litte-endian)
  39. 	("Python pyc 3.11", [0xA7, 0x0D, '\r', '\n']), // (3495i litte-endian)
  40. 	("Python pyc 3.12", [0xCB, 0x0D, '\r', '\n']), // (3531i litte-endian)
  41. 	// Python pickle object data, similarly to Perl Storage it's dangerous enough to cause code execution
  42. 	("Python Pickle v2", [0x80, 0x02]), // Protocol 2 + start of frame
  43. 	("Python Pickle v3", [0x80, 0x03]), // Protocol 3 + start of frame
  44. 	("Python Pickle v4", [0x80, 0x04, 0x95]), // Protocol 4 + start of frame
  45. 	("Python Pickle v5", [0x80, 0x05, 0x95]), // Protocol 5 + start of frame
  47. 	// https://github.com/MoarVM/MoarVM/blob/master/docs/bytecode.markdown
  48. 	("MoarVM bytecode", ['M', 'O', 'A', 'R', 'V', 'M', '\r', '\n']),
  49. 	// https://github.com/parrot/parrot/blob/master/docs/parrotbyte.pod
  50. 	("Parrot bytecode", [0xFE, 'P', 'B', 'C', '\r', '\n', 0x1A, '\n']),
  51. 	("Perl storable v0.6", ['p', 'e', 'r', 'l', '-', 's', 't', 'o', 'r', 'e']),
  52. 	("Perl storable v0.7", ['p', 's', 't', '0']),
  53. 	("Chez Scheme bytecode", [0x00, 0x00, 0x00, 0x00, 'c', 'h', 'e', 'z']),
  54. 	("NekoVM bytecode", ['N', 'E', 'K', 'O']),
  55. 	("Emacs Lisp bytecode", [';', 'E', 'L', 'C']), // Emacs lisp bytecode, if there is known false positives next 4 bytes is the version
  57. 	("OCaml", ['C', 'a', 'm', 'l', '1', '9', '9', '9']),
  59. 	("Ren'Py Archive v1", [0x78, 0x9c]),
  60. 	("Ren'Py Archive v2", ['R', 'P', 'A', '-', '2', '.', '0', ' ']),
  61. 	("Ren'Py Archive v3", ['R', 'P', 'A', '-', '3', '.', '0', ' ']),
  63. 	("Squirrel bytecode", [0xFA, 0xFA]),
  65. 	("Clang Pre-Compiled-Header", ['C', 'P', 'C', 'H']),
  66. 	// Clang Pre-Compiled-Header, followed by Info Block, see:
  67. 	// - clang/lib/Serialization/ASTWriter.cpp ASTWriter::WriteAST
  68. 	// - clang/lib/Serialization/ASTReader.cpp doesntStartWithASTFileMagic
  69. 	// Excluded from fixtures (200KB+), test with:
  70. 	//   echo > empty.h && clang -cc1 -nobuiltininc -emit-pch -o empty.h.pch empty.h
  72. 	("GCC Pre-Compiled-Header", ['g', 'p', 'c', 'h']),
  73. 	// Excluded from fixtures (1.2MB+), test with:
  74. 	//   echo > empty.h && gcc empty.h
  76. 	("GCC Rust Metadata", ['G', 'R', 'S', 'T']), // GCC Rust Metadata (*.rox)
  78. 	("Dart Kernel snapshot", [0x90, 0xab, 0xcd, 0xef]),
  79. 	// why are these 2 different, and is the C one still in use?
  80. 	// no clue, but both are in the sdk repo.
  81. 	("Dart JIT snapshot (C code)", [0xdc, 0xdc, 0xf5, 0xf5]), // Dart JIT snapshot, if done from the C code.
  82. 	("Dart JIT snapshot (Dart code)", [0xdc, 0xdc, 0xf6, 0xf6]), // Dart JIT snapshot, if done from the Dart code.
  83. ];
  84. const dos_magic: []u8 = ['M', 'Z'];
  85. const pe_magic: []u8 = ['P', 'E', 0x00, 0x00];
  86. const racket: []u8 = ['r', 'a', 'c', 'k', 'e', 't'];
  87. const zip: []u8 = ['P', 'K', 0x03, 0x04];
  88. const jar: []u8 = [0xFE, 0xCA, 0, 0];
  89. const shebang: []u8 = ['#', '!'];
  91. let found: bool = false;
  92. let json_out: io::handle = 0;
  94. fn id_blob(filename: str) (void | str | fs::error | io::error) = {
  95. 	static let buffer: [4096]u8 = [0...];
  97. 	const file = os::open(filename)?;
  98. 	defer io::close(file)!;
  100. 	if (io::read(file, buffer)? is io::EOF) {
  101. 		// empty file
  102. 		return void;
  103. 	};
  105. 	for (let i = 0z; i < len(magic); i += 1) {
  106. 		assert(len(magic[i].1) > 0);
  107. 		if (bytes::hasprefix(buffer, magic[i].1)) {
  108. 			return magic[i].0;
  109. 		};
  110. 	};
  112. 	// Special check to detect *all* Microsoft Portable Executable files
  113. 	if (bytes::hasprefix(buffer, dos_magic)) {
  114. 		const pe_offset = endian::legetu32(buffer[60..64]);
  116. 		if ((pe_offset <= 4096-4) && bytes::hasprefix(buffer[pe_offset..pe_offset+4], pe_magic)) {
  117. 			return "WinNT EXE";
  118. 		};
  119. 	};
  121. 	// detect binary escripts (PKZIP archive and BEAM supported)
  122. 	if (bytes::hasprefix(buffer, shebang)) {
  123. 		let comment = true;
  124. 		for (let i = 0z; i < 4096; i += 1) {
  125. 			if(comment) {
  126. 				if(buffer[i] == '\n') comment = false;
  128. 				continue;
  129. 			};
  131. 			if(buffer[i] == '%') {
  132. 				comment = true;
  133. 			} else {
  134. 				// First bytes after comments
  135. 				if(bytes::equal(zip, buffer[i..i+len(zip)])) return "Erlang ZIP BEAM";
  136. 				if(bytes::equal(beam, buffer[i..i+len(beam)])) return "Erlang #! BEAM";
  138. 				// source code as script
  139. 				break;
  140. 			};
  141. 		};
  142. 	};
  144. 	// Special check to detect racket bytecode
  145. 	if (bytes::hasprefix(buffer, ['#', '~'])) {
  146. 		// From src/expander/compile/write-linklet.rkt in racket:
  147. 		// - #~
  148. 		// - length-prefixed version string (ie. "\x038.5")
  149. 		// - length-prefixed virtual machine string (ie. "\x06racket")
  150. 		// - 'D' / 'B'
  151. 		// Here it verifies that the virtual machine string is racket, assuming none other is supported in the wild.
  152. 		// Racket itself only matches against '#~' which is small & only printable-ASCII, so too prone to false positives
  153. 		const version_len = buffer[2];
  154. 		const version_end = 2+version_len;
  156. 		const racket_len = buffer[version_end+1];
  157. 		const racket_start = version_end+2;
  159. 		if(bytes::equal(racket, buffer[racket_start..racket_start+racket_len])) {
  160. 			return "Racket";
  161. 		};
  162. 	};
  164. 	if (bytes::hasprefix(buffer, zip)) {
  165. 		if(bytes::equal(jar, buffer[0x27..0x2B])) {
  166. 			return "Java JAR";
  167. 		};
  168. 	};
  170. 	return void;
  171. };
  173. @test fn id_blob() void = {
  174. 	const sources = [
  175. 		"test/fixtures/empty",
  176. 		"test/fixtures/empty.dts",
  177. 		"test/fixtures/hello-dart.dart",
  178. 		"test/fixtures/hello-ocaml.ml",
  179. 		"test/fixtures/hello-racket.rkt",
  180. 		"test/fixtures/hello.1",
  181. 		"test/fixtures/hello.c",
  182. 		"test/fixtures/hello.cs",
  183. 		"test/fixtures/hello.el",
  184. 		"test/fixtures/hello.erl",
  185. 		"test/fixtures/hello.erl.escript",
  186. 		"test/fixtures/hello.java",
  187. 		"test/fixtures/hello.lua",
  188. 		"test/fixtures/hello.neko",
  189. 		"test/fixtures/hello.nqp",
  190. 		"test/fixtures/hello.nut",
  191. 		"test/fixtures/hello.pir",
  192. 		"test/fixtures/hello.py",
  193. 		"test/fixtures/hello.wat",
  194. 		"test/fixtures/perl_storage.pm",
  195. 	];
  197. 	for (let i = 0z; i < len(sources); i += 1) {
  198. 		match(id_blob(sources[i])!) {
  199. 		case void =>
  200. 			continue;
  201. 		case let s: str =>
  202. 			fmt::fatalf(
  203. 				"deblob: error: id_blob({}) got wrongly detected as: {}",
  204. 				sources[i], s
  205. 			);
  206. 		};
  207. 	};
  209. 	const blobs = [
  210. 		("Racket", "test/fixtures/compiled/hello-racket_rkt.zo"),
  211. 		("DTB", "test/fixtures/empty.dtb"),
  212. 		("ELF", "test/fixtures/hello"),
  213. 		("Dart Kernel snapshot", "test/fixtures/hello-dart.dill"),
  214. 		("Dart JIT snapshot (Dart code)", "test/fixtures/hello-dart.jit"),
  215. 		("Unix ar(1)", "test/fixtures/hello-ocaml.a"),
  216. 		("OCaml", "test/fixtures/hello-ocaml.cma"),
  217. 		("OCaml", "test/fixtures/hello-ocaml.cmi"),
  218. 		("OCaml", "test/fixtures/hello-ocaml.cmo"),
  219. 		("OCaml", "test/fixtures/hello-ocaml.cmx"),
  220. 		("OCaml", "test/fixtures/hello-ocaml.cmxa"),
  221. 		("ELF", "test/fixtures/hello-ocaml.o"),
  222. 		("Unix ar(1)", "test/fixtures/hello.a"),
  223. 		("Erlang FOR1 BEAM", "test/fixtures/hello.beam"),
  224. 		("Erlang #! BEAM", "test/fixtures/hello.beam.escript"),
  225. 		("Java .class / MachO exec", "test/fixtures/hello.class"),
  226. 		("Squirrel bytecode", "test/fixtures/hello.cnut"),
  227. 		("Emacs Lisp bytecode", "test/fixtures/hello.elc"),
  228. 		("WinNT EXE", "test/fixtures/hello.exe"),
  229. 		("Java JAR", "test/fixtures/hello.jar"),
  230. 		("Lua bytecode", "test/fixtures/hello.luac53"),
  231. 		("Lua bytecode", "test/fixtures/hello.luac54"),
  232. 		("NekoVM bytecode", "test/fixtures/hello.n"),
  233. 		("MoarVM bytecode", "test/fixtures/hello.nqp.moarvm"),
  234. 		("ELF", "test/fixtures/hello.o"),
  235. 		("Parrot bytecode", "test/fixtures/hello.pir.pbc"),
  236. 		("Wasm", "test/fixtures/hello.wasm"),
  237. 		("WinNT EXE", "test/fixtures/monodx.dll"),
  238. 		("Perl storable v0.7", "test/fixtures/perl_storage.pst"),
  239. 		("Python Pickle v4", "test/fixtures/pickle/hello.4.pickle"),
  240. 		("Python Pickle v5", "test/fixtures/pickle/hello.5.pickle"),
  241. 		("Apple PEF", "test/fixtures/qemu_vga.ndrv"),
  242. 		//("", "test/fixtures/option.rom"),
  243. 	];
  245. 	for (let i = 0z; i < len(blobs); i += 1) {
  246. 		match(id_blob(blobs[i].1)!) {
  247. 		case void =>
  248. 			fmt::fatalf(
  249. 				"deblob: error: id_blob({}) didn't got detected as: {}",
  250. 				blobs[i].1, blobs[i].0
  251. 			);
  252. 		case let s: str =>
  253. 			if(s != blobs[i].0)
  254. 			{
  255. 				fmt::fatalf(
  256. 					"deblob: error: id_blob({}) got identified as \"{}\" instead of \"{}\"",
  257. 					blobs[i].1, s, blobs[i].0
  258. 				);
  259. 			};
  260. 		};
  261. 	};
  262. };
  264. fn is_excluded(filename: str) bool = {
  265. 	for (let i = 0z; i < len(excludes); i += 1) {
  266. 		if (fnmatch::fnmatch(excludes[i], filename, fnmatch::flag::NONE)) {
  267. 			return true;
  268. 		};
  269. 	};
  270. 	return false;
  271. };
  273. fn append_action(action: str, filename: str, format: str) void = {
  274. 	if(!json) return;
  276. 	let obj = json::object { ... };
  278. 	json::put(&obj, "action", action);
  279. 	defer json::take(&obj, "action");
  280. 	json::put(&obj, "path", filename);
  281. 	defer json::take(&obj, "path");
  282. 	json::put(&obj, "format", format);
  283. 	defer json::take(&obj, "format");
  285. 	let obj_s = json::dumpstr(obj);
  286. 	defer free(obj_s);
  288. 	static let first_obj: bool = true;
  289. 	if(first_obj)
  290. 	{
  291. 		fmt::fprintf(json_out, "\n\t{}", obj_s)!;
  292. 		first_obj = false;
  293. 	}
  294. 	else
  295. 	{
  296. 		fmt::fprintf(json_out, ",\n\t{}", obj_s)!;
  297. 	};
  298. };
  300. fn check_dir(dirname: str) (void | errors::invalid | io::error) = {
  301. 	const iter = match (os::iter(dirname)) {
  302. 	case let iter: *fs::iterator =>
  303. 		yield iter;
  304. 	case let err: fs::error =>
  305. 		fmt::errorfln("deblob: error: Failed walking directory '{}': {}", dirname, fs::strerror(err))!;
  306. 		return errors::invalid;
  307. 	};
  308. 	defer fs::finish(iter);
  310. 	for (true) {
  311. 		const ent: fs::dirent = match (fs::next(iter)) {
  312. 		case let ent: fs::dirent =>
  313. 			yield ent;
  314. 		case let e: fs::error =>
  315. 			fmt::errorfln("deblob: error: Failed walking directory '{}': {}", dirname, fs::strerror(e))?;
  316. 			break;
  317. 		case done =>
  318. 			break;
  319. 		};
  321. 		const filename_path = path::init(dirname, ent.name)!;
  322. 		const filename = path::string(&filename_path);
  324. 		if (fs::isdir(ent.ftype)) {
  325. 			check_dir(filename)?;
  326. 		} else if(fs::isfile(ent.ftype)) {
  327. 			const blob_id = match (id_blob(filename)) {
  328. 			case void =>
  329. 				continue;
  330. 			case let s: str =>
  331. 				yield s;
  332. 			case let err: fs::error =>
  333. 				fmt::errorfln("deblob: error: Failed opening {}: {}",
  334. 					filename, fs::strerror(err))!;
  335. 				continue;
  336. 			case let err: io::error =>
  337. 				fmt::errorfln("deblob: error: Failed reading {}: {}",
  338. 					filename, io::strerror(err))!;
  339. 				continue;
  340. 			};
  342. 			if (is_excluded(filename)) {
  343. 				append_action("ignoring", filename, blob_id);
  344. 				fmt::printfln("ignoring {}:\t{}", blob_id, filename)!;
  345. 				continue;
  346. 			};
  348. 			found = true;
  350. 			if (noop) {
  351. 				append_action("detected", filename, blob_id);
  352. 				fmt::printfln("detected {}:\t{}", blob_id, filename)!;
  353. 				continue;
  354. 			};
  356. 			append_action("removing", filename, blob_id);
  357. 			fmt::printfln("removing {}:\t{}", blob_id, filename)!;
  359. 			match (os::remove(filename)) {
  360. 			case void =>
  361. 				continue;
  362. 			case let e: fs::error =>
  363. 				fmt::errorfln("deblob: error: Failed removing file '{}':\t{}",
  364. 					filename, fs::strerror(e))!;
  365. 			};
  366. 		} else {
  367. 			// ignore non-(dir/regular-file) like symlinks, blocks, fifo, …
  368. 			continue;
  369. 		};
  370. 	};
  371. };
  373. @test fn check_dir() void = {
  374. 	const dirname = "test/check_dir-fixtures";
  376. 	const files_before = match (os::readdir(dirname)) {
  377. 	case let d: []fs::dirent =>
  378. 		yield d;
  379. 	case let e: fs::error =>
  380. 		fmt::fatalf("deblob: error: os::readdir({}): {}", dirname, fs::strerror(e));
  381. 	};
  382. 	assert(len(files_before) == 60);
  384. 	const ret = check_dir(dirname);
  385. 	assert(ret is void);
  387. 	const files_after = match (os::readdir(dirname)) {
  388. 	case let d: []fs::dirent =>
  389. 		yield d;
  390. 	case let e: fs::error =>
  391. 		fmt::fatalf("deblob: error: os::readdir({}): {}", dirname, fs::strerror(e));
  392. 	};
  393. 	assert(len(files_after) == 30);
  394. };
  396. export fn main() void = {
  397. 	const cmd = getopt::parse(os::args,
  398. 		"Remove binary executable files",
  399. 		('c', "Return error if any non-excluded blobs were found"),
  400. 		('e', "NAME", "Exclude filename from removal (defaults to none)"),
  401. 		('d', "PATH", "Set working directory (default to current dir)"),
  402. 		('j', "PATH", "JSON output file"),
  403. 		('n', "No actual removal, only scan and log"),
  404. 	);
  405. 	defer getopt::finish(&cmd);
  406. 	defer free(excludes);
  408. 	let opt_d = "";
  409. 	let json_out_path = "";
  411. 	for (let i = 0z; i < len(cmd.opts); i += 1) {
  412. 		const opt = cmd.opts[i];
  413. 		switch (opt.0) {
  414. 		case 'c' =>
  415. 			check = true;
  416. 		case 'e' =>
  417. 			append(excludes, opt.1);
  418. 		case 'd' =>
  419. 			opt_d = opt.1;
  420. 		case 'n' =>
  421. 			noop = true;
  422. 		case 'j' =>
  423. 			json = true;
  424. 			json_out_path = opt.1;
  425. 		case =>
  426. 			fmt::fatalf("deblob: error: Unhandled option -{}", opt.0);
  427. 		};
  428. 	};
  430. 	if(json_out_path != "")
  431. 	{
  432. 		json_out = match (os::create(json_out_path, fs::mode::USER_RW | fs::mode::GROUP_R | fs::mode::OTHER_R)) {
  433. 		case let f: io::file =>
  434. 			yield f;
  435. 		case let e: fs::error =>
  436. 			fmt::fatalf("deblob: error: Failed creating/opening file '{}' for JSON output: {}", json_out_path, fs::strerror(e));
  437. 		};
  439. 		fmt::fprint(json_out, "[")!;
  440. 	};
  442. 	if(opt_d != "")
  443. 	{
  444. 		match (os::chdir(opt_d)) {
  445. 		case let e: fs::error =>
  446. 			fmt::fatalf("deblob: error: Failed changing current directory to '{}': {}", opt_d, fs::strerror(e));
  447. 		case void =>
  448. 			void;
  449. 		};
  450. 	};
  452. 	fmt::println(":: Checking for blobs")!;
  454. 	const ret = check_dir(".");
  456. 	fmt::println(":: Done checking for blobs")!;
  458. 	if(json_out_path != "")
  459. 	{
  460. 		fmt::fprint(json_out, "\n]")!;
  462. 		match(io::close(json_out)) {
  463. 		case void =>
  464. 			void;
  465. 		case let e: io::error =>
  466. 			fmt::fatalf("deblob: error: Failed closing JSON output file '{}': {}", json_out_path, io::strerror(e));
  467. 		};
  468. 	};
  470. 	match (ret) {
  471. 	case void =>
  472. 		if(check && found) os::exit(2);
  474. 		os::exit(0);
  475. 	case errors::invalid =>
  476. 		os::exit(1);
  477. 	case let e: io::error =>
  478. 		fmt::errorfln("deblob: error: I/O error while traversing directories: {}", io::strerror(e))!;
  479. 		os::exit(1);
  480. 	};
  481. };