logo

deblob

remove binary executables from a directory git clone https://anongit.hacktivis.me/git/deblob.git/

main.ha (16165B)

    

  1. // Copyright © 2019 deblob Authors <https://hacktivis.me/projects/deblob>
  2. // SPDX-License-Identifier: BSD-3-Clause
  3. use bytes;
  4. use encoding::json;
  5. use endian;
  6. use errors;
  7. use fmt;
  8. use fnmatch;
  9. use fs;
  10. use getopt;
  11. use io;
  12. use os;
  13. use path;
  14. use strings;
  16. let excludes: []str = [];
  17. let noop: bool = false;
  18. let check: bool = false;
  19. let json: bool = false;
  21. const beam: []u8 = ['F', 'O', 'R', '1']; // Erlang BEAM
  22. const magic: [_](str, []u8) = [
  23. 	("ELF", [0x7F, 'E', 'L', 'F']),
  24. 	("Unix ar(1)", ['!', '<', 'a', 'r', 'c', 'h', '>', '\n']),
  25. 	("PC-BIOS", [0x55, 0xAA]),
  26. 	("Erlang FOR1 BEAM", ['F', 'O', 'R', '1']),
  27. 	("Java .class / Mach-O exec", [0xCA, 0xFE, 0xBA, 0xBE]),
  28. 	("Mach-O exec", [0xCF, 0xFA, 0xED, 0xFE]),
  29. 	("WinNT EXE", ['M', 'Z', 0x90, 0x00, 0x03, 0x00, 0x00, 0x00]), // Partial MSDOS stub header (ECMA-335 Common Language Infrastructure)
  30. 	("Lua bytecode", [0x1B, 'L', 'u', 'a']),
  31. 	("Wasm", [0x00, 'a', 's', 'm']),
  32. 	("Apple PEF", ['J', 'o', 'y', '!', 'p', 'e', 'f', 'f']), // Apple Preferred Executable Format
  33. 	("DTB", [0xD0, 0x0D, 0xFE, 0xED]), // Device Tree Blob (OpenFirmware, u-boot, …)
  35. 	// Python *.pyc bytecode magic numbers (defined in importlib/_bootstrap_external.py)
  36. 	("Python pyc 2.7", [0x03, 0xF3, '\r', '\n']), // (62211i little-endian)
  37. 	("Python pyc 3.8", [0x55, 0x0D, '\r', '\n']), // (3413i little-endian)
  38. 	("Python pyc 3.9", [0x61, 0x0D, '\r', '\n']), // (3425i little-endian)
  39. 	("Python pyc 3.10", [0x6F, 0x0D, '\r', '\n']), // (3439i little-endian)
  40. 	("Python pyc 3.11", [0xA7, 0x0D, '\r', '\n']), // (3495i little-endian)
  41. 	("Python pyc 3.12", [0xCB, 0x0D, '\r', '\n']), // (3531i little-endian)
  42. 	("Python pyc 3.13", [0xF3, 0x0D, '\r', '\n']), // (3571i little-endian)
  43. 	("Python pyc 3.14", [0x2B, 0x0E, '\r', '\n']), // (3627i little-endian)
  44. 	// Python pickle object data, similarly to Perl Storage it's dangerous enough to cause code execution
  45. 	("Python Pickle v2", [0x80, 0x02]), // Protocol 2 + start of frame
  46. 	("Python Pickle v3", [0x80, 0x03]), // Protocol 3 + start of frame
  47. 	("Python Pickle v4", [0x80, 0x04, 0x95]), // Protocol 4 + start of frame
  48. 	("Python Pickle v5", [0x80, 0x05, 0x95]), // Protocol 5 + start of frame
  50. 	// https://github.com/MoarVM/MoarVM/blob/master/docs/bytecode.markdown
  51. 	("MoarVM bytecode", ['M', 'O', 'A', 'R', 'V', 'M', '\r', '\n']),
  52. 	// https://github.com/parrot/parrot/blob/master/docs/parrotbyte.pod
  53. 	("Parrot bytecode", [0xFE, 'P', 'B', 'C', '\r', '\n', 0x1A, '\n']),
  54. 	("Perl storable v0.6", ['p', 'e', 'r', 'l', '-', 's', 't', 'o', 'r', 'e']),
  55. 	("Perl storable v0.7", ['p', 's', 't', '0']),
  56. 	("Chez Scheme bytecode", [0x00, 0x00, 0x00, 0x00, 'c', 'h', 'e', 'z']),
  57. 	("NekoVM bytecode", ['N', 'E', 'K', 'O']),
  58. 	("Emacs Lisp bytecode", [';', 'E', 'L', 'C']), // Emacs lisp bytecode, if there is known false positives next 4 bytes is the version
  60. 	("OCaml", ['C', 'a', 'm', 'l', '1', '9', '9', '9']),
  62. 	("Ren'Py Archive v1", [0x78, 0x9c]),
  63. 	("Ren'Py Archive v2", ['R', 'P', 'A', '-', '2', '.', '0', ' ']),
  64. 	("Ren'Py Archive v3", ['R', 'P', 'A', '-', '3', '.', '0', ' ']),
  66. 	("Squirrel bytecode", [0xFA, 0xFA]),
  68. 	("Clang Pre-Compiled-Header", ['C', 'P', 'C', 'H']),
  69. 	// Clang Pre-Compiled-Header, followed by Info Block, see:
  70. 	// - clang/lib/Serialization/ASTWriter.cpp ASTWriter::WriteAST
  71. 	// - clang/lib/Serialization/ASTReader.cpp doesntStartWithASTFileMagic
  72. 	// Excluded from fixtures (200KB+), test with:
  73. 	//   echo > empty.h && clang -cc1 -nobuiltininc -emit-pch -o empty.h.pch empty.h
  75. 	("GCC Pre-Compiled-Header", ['g', 'p', 'c', 'h']),
  76. 	// Excluded from fixtures (1.2MB+), test with:
  77. 	//   echo > empty.h && gcc empty.h
  79. 	("GCC Rust Metadata", ['G', 'R', 'S', 'T']), // GCC Rust Metadata (*.rox)
  81. 	("Dart Kernel snapshot", [0x90, 0xab, 0xcd, 0xef]),
  82. 	// why are these 2 different, and is the C one still in use?
  83. 	// no clue, but both are in the sdk repo.
  84. 	("Dart JIT snapshot (C code)", [0xdc, 0xdc, 0xf5, 0xf5]), // Dart JIT snapshot, if done from the C code.
  85. 	("Dart JIT snapshot (Dart code)", [0xdc, 0xdc, 0xf6, 0xf6]), // Dart JIT snapshot, if done from the Dart code.
  86. ];
  87. const dos_magic: []u8 = ['M', 'Z'];
  88. const pe_magic: []u8 = ['P', 'E', 0x00, 0x00];
  89. const racket: []u8 = ['r', 'a', 'c', 'k', 'e', 't'];
  90. const zip: []u8 = ['P', 'K', 0x03, 0x04];
  91. const jar: []u8 = [0xFE, 0xCA, 0, 0];
  92. const shebang: []u8 = ['#', '!'];
  93. const meta_inf_: []u8 = ['M', 'E', 'T', 'A', '-', 'I', 'N', 'F', '/'];
  95. let found: bool = false;
  96. let json_out: io::handle = 0;
  98. fn id_blob(filename: str) (void | str | fs::error | io::error) = {
  99. 	static let buffer: [4096]u8 = [0...];
  101. 	const file = os::open(filename)?;
  102. 	defer io::close(file)!;
  104. 	if (io::read(file, buffer)? is io::EOF) {
  105. 		// empty file
  106. 		return void;
  107. 	};
  109. 	for (let i = 0z; i < len(magic); i += 1) {
  110. 		assert(len(magic[i].1) > 0);
  111. 		if (bytes::hasprefix(buffer, magic[i].1)) {
  112. 			return magic[i].0;
  113. 		};
  114. 	};
  116. 	// Special check to detect *all* Microsoft Portable Executable files
  117. 	if (bytes::hasprefix(buffer, dos_magic)) {
  118. 		const pe_offset = endian::legetu32(buffer[60..64]);
  120. 		if ((pe_offset <= 4096-4) && bytes::hasprefix(buffer[pe_offset..pe_offset+4], pe_magic)) {
  121. 			return "WinNT EXE";
  122. 		};
  123. 	};
  125. 	// detect binary escripts (PKZIP archive and BEAM supported)
  126. 	if (bytes::hasprefix(buffer, shebang)) {
  127. 		let comment = true;
  128. 		for (let i = 0z; i < 4096; i += 1) {
  129. 			if(comment) {
  130. 				if(buffer[i] == '\n') comment = false;
  132. 				continue;
  133. 			};
  135. 			if(buffer[i] == '%') {
  136. 				comment = true;
  137. 			} else {
  138. 				// First bytes after comments
  139. 				if(bytes::equal(zip, buffer[i..i+len(zip)])) return "Erlang ZIP BEAM";
  140. 				if(bytes::equal(beam, buffer[i..i+len(beam)])) return "Erlang #! BEAM";
  142. 				// source code as script
  143. 				break;
  144. 			};
  145. 		};
  146. 	};
  148. 	// Special check to detect racket bytecode
  149. 	if (bytes::hasprefix(buffer, ['#', '~'])) {
  150. 		// From src/expander/compile/write-linklet.rkt in racket:
  151. 		// - #~
  152. 		// - length-prefixed version string (ie. "\x038.5")
  153. 		// - length-prefixed virtual machine string (ie. "\x06racket")
  154. 		// - 'D' / 'B'
  155. 		// Here it verifies that the virtual machine string is racket, assuming none other is supported in the wild.
  156. 		// Racket itself only matches against '#~' which is small & only printable-ASCII, so too prone to false positives
  157. 		const version_len = buffer[2];
  158. 		const version_end = 2+version_len;
  160. 		const racket_len = buffer[version_end+1];
  161. 		const racket_start = version_end+2;
  163. 		if(bytes::equal(racket, buffer[racket_start..racket_start+racket_len])) {
  164. 			return "Racket";
  165. 		};
  166. 	};
  168. 	if (bytes::hasprefix(buffer, zip)) {
  169. 		// Check first filename
  170. 		// Optional in JAR and probably doesn't have to be the first filename
  171. 		// but it's how it's usually done
  172. 		//
  173. 		// So far seen either META-INF/ and \xFE\xCA\x00\x00 in extra or just META-INF/MANIFEST.MF
  175. 		const fname_len = endian::legetu16(buffer[0x1A..0x1C]);
  176. 		if(fname_len < 256)
  177. 		{
  178. 			const fname_start = 0x1Eu16;
  179. 			if(fname_len >= 9 && bytes::equal(meta_inf_, buffer[fname_start..fname_start+9])) {
  180. 				return "Java JAR";
  181. 			};
  183. 			const extra_start = fname_start+fname_len;
  184. 			const extra_len = endian::legetu16(buffer[0x1D..0x1F]);
  185. 			if(extra_len == 4 && bytes::equal(jar, buffer[extra_start..extra_start+extra_len]))
  186. 			{
  187. 				return "Java JAR";
  188. 			};
  189. 		};
  190. 	};
  192. 	return void;
  193. };
  195. @test fn id_blob() void = {
  196. 	const sources = [
  197. 		"test/fixtures/empty",
  198. 		"test/fixtures/empty.dts",
  199. 		"test/fixtures/hello-dart.dart",
  200. 		"test/fixtures/hello-ocaml.ml",
  201. 		"test/fixtures/hello-racket.rkt",
  202. 		"test/fixtures/hello.1",
  203. 		"test/fixtures/hello.c",
  204. 		"test/fixtures/hello.cs",
  205. 		"test/fixtures/hello.el",
  206. 		"test/fixtures/hello.erl",
  207. 		"test/fixtures/hello.erl.escript",
  208. 		"test/fixtures/hello.java",
  209. 		"test/fixtures/hello.lua",
  210. 		"test/fixtures/hello.neko",
  211. 		"test/fixtures/hello.nqp",
  212. 		"test/fixtures/hello.nut",
  213. 		"test/fixtures/hello.pir",
  214. 		"test/fixtures/hello.py",
  215. 		"test/fixtures/hello.wat",
  216. 		"test/fixtures/perl_storage.pm",
  217. 	];
  219. 	for (let i = 0z; i < len(sources); i += 1) {
  220. 		match(id_blob(sources[i])!) {
  221. 		case void =>
  222. 			continue;
  223. 		case let s: str =>
  224. 			fmt::fatalf(
  225. 				"deblob: error: id_blob({}) got wrongly detected as: {}",
  226. 				sources[i], s
  227. 			);
  228. 		};
  229. 	};
  231. 	const blobs = [
  232. 		("Racket", "test/fixtures/compiled/hello-racket_rkt.zo"),
  233. 		("DTB", "test/fixtures/empty.dtb"),
  234. 		("ELF", "test/fixtures/hello"),
  235. 		("Dart Kernel snapshot", "test/fixtures/hello-dart.dill"),
  236. 		("Dart JIT snapshot (Dart code)", "test/fixtures/hello-dart.jit"),
  237. 		("Unix ar(1)", "test/fixtures/hello-ocaml.a"),
  238. 		("OCaml", "test/fixtures/hello-ocaml.cma"),
  239. 		("OCaml", "test/fixtures/hello-ocaml.cmi"),
  240. 		("OCaml", "test/fixtures/hello-ocaml.cmo"),
  241. 		("OCaml", "test/fixtures/hello-ocaml.cmx"),
  242. 		("OCaml", "test/fixtures/hello-ocaml.cmxa"),
  243. 		("ELF", "test/fixtures/hello-ocaml.o"),
  244. 		("Unix ar(1)", "test/fixtures/hello.a"),
  245. 		("Erlang FOR1 BEAM", "test/fixtures/hello.beam"),
  246. 		("Erlang #! BEAM", "test/fixtures/hello.beam.escript"),
  247. 		("Java .class / Mach-O exec", "test/fixtures/hello.class"),
  248. 		("Squirrel bytecode", "test/fixtures/hello.cnut"),
  249. 		("Emacs Lisp bytecode", "test/fixtures/hello.elc"),
  250. 		("WinNT EXE", "test/fixtures/hello.exe"),
  251. 		("Java JAR", "test/fixtures/hello.jar"),
  252. 		("Lua bytecode", "test/fixtures/hello.luac53"),
  253. 		("Lua bytecode", "test/fixtures/hello.luac54"),
  254. 		("NekoVM bytecode", "test/fixtures/hello.n"),
  255. 		("MoarVM bytecode", "test/fixtures/hello.nqp.moarvm"),
  256. 		("ELF", "test/fixtures/hello.o"),
  257. 		("Parrot bytecode", "test/fixtures/hello.pir.pbc"),
  258. 		("Wasm", "test/fixtures/hello.wasm"),
  259. 		("WinNT EXE", "test/fixtures/monodx.dll"),
  260. 		("Perl storable v0.7", "test/fixtures/perl_storage.pst"),
  261. 		("Python Pickle v4", "test/fixtures/pickle/hello.4.pickle"),
  262. 		("Python Pickle v5", "test/fixtures/pickle/hello.5.pickle"),
  263. 		("Apple PEF", "test/fixtures/qemu_vga.ndrv"),
  264. 		//("", "test/fixtures/option.rom"),
  265. 		("Mach-O exec", "test/fixtures/macos-arm64.o"),
  266. 	];
  268. 	for (let i = 0z; i < len(blobs); i += 1) {
  269. 		match(id_blob(blobs[i].1)!) {
  270. 		case void =>
  271. 			fmt::fatalf(
  272. 				"deblob: error: id_blob({}) didn't got detected as: {}",
  273. 				blobs[i].1, blobs[i].0
  274. 			);
  275. 		case let s: str =>
  276. 			if(s != blobs[i].0)
  277. 			{
  278. 				fmt::fatalf(
  279. 					"deblob: error: id_blob({}) got identified as \"{}\" instead of \"{}\"",
  280. 					blobs[i].1, s, blobs[i].0
  281. 				);
  282. 			};
  283. 		};
  284. 	};
  285. };
  287. fn is_excluded(filename: str) bool = {
  288. 	for (let i = 0z; i < len(excludes); i += 1) {
  289. 		if (fnmatch::fnmatch(excludes[i], filename, fnmatch::flag::NONE)) {
  290. 			return true;
  291. 		};
  292. 	};
  293. 	return false;
  294. };
  296. fn append_action(action: str, filename: str, format: str) (void | nomem) = {
  297. 	if(!json) return;
  299. 	let obj = json::object { ... };
  301. 	json::put(&obj, "action", action)?;
  302. 	defer json::take(&obj, "action");
  303. 	json::put(&obj, "path", filename)?;
  304. 	defer json::take(&obj, "path");
  305. 	json::put(&obj, "format", format)?;
  306. 	defer json::take(&obj, "format");
  308. 	let obj_s = json::dumpstr(obj);
  309. 	defer free(obj_s);
  311. 	static let first_obj: bool = true;
  312. 	if(first_obj)
  313. 	{
  314. 		fmt::fprintf(json_out, "\n\t{}", obj_s)!;
  315. 		first_obj = false;
  316. 	}
  317. 	else
  318. 	{
  319. 		fmt::fprintf(json_out, ",\n\t{}", obj_s)!;
  320. 	};
  321. };
  323. fn check_dir(dirname: str) (void | nomem | errors::invalid | io::error) = {
  324. 	const iter = match (os::iter(dirname)) {
  325. 	case let iter: *fs::iterator =>
  326. 		yield iter;
  327. 	case let err: fs::error =>
  328. 		fmt::errorfln("deblob: error: Failed walking directory '{}': {}", dirname, fs::strerror(err))!;
  329. 		return errors::invalid;
  330. 	};
  331. 	defer fs::finish(iter);
  333. 	for (true) {
  334. 		const ent: fs::dirent = match (fs::next(iter)) {
  335. 		case let ent: fs::dirent =>
  336. 			yield ent;
  337. 		case let e: fs::error =>
  338. 			fmt::errorfln("deblob: error: Failed walking directory '{}': {}", dirname, fs::strerror(e))?;
  339. 			break;
  340. 		case done =>
  341. 			break;
  342. 		};
  344. 		const filename_path = path::init(dirname, ent.name)!;
  345. 		const filename = path::string(&filename_path);
  347. 		if (fs::isdir(ent.ftype)) {
  348. 			check_dir(filename)?;
  349. 		} else if(fs::isfile(ent.ftype)) {
  350. 			const blob_id = match (id_blob(filename)) {
  351. 			case void =>
  352. 				continue;
  353. 			case let s: str =>
  354. 				yield s;
  355. 			case let err: fs::error =>
  356. 				fmt::errorfln("deblob: error: Failed opening {}: {}",
  357. 					filename, fs::strerror(err))!;
  358. 				continue;
  359. 			case let err: io::error =>
  360. 				fmt::errorfln("deblob: error: Failed reading {}: {}",
  361. 					filename, io::strerror(err))!;
  362. 				continue;
  363. 			};
  365. 			if (is_excluded(filename)) {
  366. 				append_action("ignoring", filename, blob_id)?;
  367. 				fmt::printfln("ignoring {}:\t{}", blob_id, filename)!;
  368. 				continue;
  369. 			};
  371. 			found = true;
  373. 			if (noop) {
  374. 				append_action("detected", filename, blob_id)?;
  375. 				fmt::printfln("detected {}:\t{}", blob_id, filename)!;
  376. 				continue;
  377. 			};
  379. 			append_action("removing", filename, blob_id)?;
  380. 			fmt::printfln("removing {}:\t{}", blob_id, filename)!;
  382. 			match (os::remove(filename)) {
  383. 			case void =>
  384. 				continue;
  385. 			case let e: fs::error =>
  386. 				fmt::errorfln("deblob: error: Failed removing file '{}':\t{}",
  387. 					filename, fs::strerror(e))!;
  388. 			};
  389. 		} else {
  390. 			// ignore non-(dir/regular-file) like symlinks, blocks, fifo, …
  391. 			continue;
  392. 		};
  393. 	};
  394. };
  396. @test fn check_dir() void = {
  397. 	const dirname = "test/check_dir-fixtures";
  399. 	const files_before = match (os::readdir(dirname)) {
  400. 	case let d: []fs::dirent =>
  401. 		yield d;
  402. 	case let e: fs::error =>
  403. 		fmt::fatalf("deblob: error: os::readdir({}): {}", dirname, fs::strerror(e));
  404. 	case nomem =>
  405. 		fmt::fatalf("deblob: error: os::readdir({}): Out of Memory", dirname);
  406. 	};
  408. 	const files_before_exp = 63z;
  409. 	if(len(files_before) != files_before_exp)
  410. 	{
  411. 		fmt::fatalf("deblob: expected {} in files_before, got {}\n", files_before_exp, len(files_before));
  412. 	};
  414. 	const ret = check_dir(dirname);
  415. 	assert(ret is void);
  417. 	const files_after = match (os::readdir(dirname)) {
  418. 	case let d: []fs::dirent =>
  419. 		yield d;
  420. 	case let e: fs::error =>
  421. 		fmt::fatalf("deblob: error: os::readdir({}): {}", dirname, fs::strerror(e));
  422. 	case nomem =>
  423. 		fmt::fatalf("deblob: error: os::readdir({}): Out of Memory", dirname);
  424. 	};
  426. 	const files_after_exp = 31z;
  427. 	if(len(files_after) != files_after_exp)
  428. 	{
  429. 		fmt::fatalf("deblob: expected {} in files_before, got {}\n", files_after_exp, len(files_after));
  430. 	};
  431. };
  433. export fn main() void = {
  434. 	const cmd = getopt::parse(os::args,
  435. 		"Remove binary executable files",
  436. 		('c', "Return error if any non-excluded blobs were found"),
  437. 		('e', "NAME", "Exclude filename from removal (defaults to none)"),
  438. 		('d', "PATH", "Set working directory (default to current dir)"),
  439. 		('j', "PATH", "JSON output file"),
  440. 		('n', "No actual removal, only scan and log"),
  441. 	);
  442. 	defer getopt::finish(&cmd);
  443. 	defer free(excludes);
  445. 	let opt_d = "";
  446. 	let json_out_path = "";
  448. 	for (let i = 0z; i < len(cmd.opts); i += 1) {
  449. 		const opt = cmd.opts[i];
  450. 		switch (opt.0) {
  451. 		case 'c' =>
  452. 			check = true;
  453. 		case 'e' =>
  454. 			append(excludes, opt.1)!;
  455. 		case 'd' =>
  456. 			opt_d = opt.1;
  457. 		case 'n' =>
  458. 			noop = true;
  459. 		case 'j' =>
  460. 			json = true;
  461. 			json_out_path = opt.1;
  462. 		case =>
  463. 			fmt::fatalf("deblob: error: Unhandled option -{}", opt.0);
  464. 		};
  465. 	};
  467. 	if(json_out_path != "")
  468. 	{
  469. 		json_out = match (os::create(json_out_path, fs::mode::USER_RW | fs::mode::GROUP_R | fs::mode::OTHER_R)) {
  470. 		case let f: io::file =>
  471. 			yield f;
  472. 		case let e: fs::error =>
  473. 			fmt::fatalf("deblob: error: Failed creating/opening file '{}' for JSON output: {}", json_out_path, fs::strerror(e));
  474. 		};
  476. 		fmt::fprint(json_out, "[")!;
  477. 	};
  479. 	if(opt_d != "")
  480. 	{
  481. 		match (os::chdir(opt_d)) {
  482. 		case let e: fs::error =>
  483. 			fmt::fatalf("deblob: error: Failed changing current directory to '{}': {}", opt_d, fs::strerror(e));
  484. 		case void =>
  485. 			void;
  486. 		};
  487. 	};
  489. 	fmt::println(":: Checking for blobs")!;
  491. 	const ret = check_dir(".");
  493. 	fmt::println(":: Done checking for blobs")!;
  495. 	if(json_out_path != "")
  496. 	{
  497. 		fmt::fprint(json_out, "\n]")!;
  499. 		match(io::close(json_out)) {
  500. 		case void =>
  501. 			void;
  502. 		case let e: io::error =>
  503. 			fmt::fatalf("deblob: error: Failed closing JSON output file '{}': {}", json_out_path, io::strerror(e));
  504. 		};
  505. 	};
  507. 	match (ret) {
  508. 	case void =>
  509. 		if(check && found) os::exit(2);
  511. 		os::exit(0);
  512. 	case errors::invalid =>
  513. 		os::exit(1);
  514. 	case nomem =>
  515. 		fmt::fatal("deblob: error: Out of Memory");
  516. 	case let e: io::error =>
  517. 		fmt::errorfln("deblob: error: I/O error while traversing directories: {}", io::strerror(e))!;
  518. 		os::exit(1);
  519. 	};
  520. };