ref: e2e50a8e5a74452672f9418bcdfe4de10f600ed0
parent: 6c2f8fd5e4f0e0be44495a5526930fa3760cd87e
author: henesy <devnull@localhost>
date: Wed Oct 2 18:55:44 EDT 2019
add zipfs(4) and getzip(1) from https://github.com/mjl-/zipfs
--- a/appl/cmd/mkfile
+++ b/appl/cmd/mkfile
@@ -21,6 +21,7 @@
spki\
ssh\
usb\
+ zip\
TARG=\
9660srv.dis\
@@ -34,7 +35,7 @@
auhdr.dis\
basename.dis\
bind.dis\
- # bit2gif.dis\
+ bit2gif.dis\
bytes.dis\
cal.dis\
calc.dis\
--- /dev/null
+++ b/appl/cmd/zip/getzip.b
@@ -1,0 +1,156 @@
+implement Getzip;
+
+include "sys.m";
+ sys: Sys;
+ sprint: import sys;
+include "draw.m";
+include "arg.m";
+include "string.m";
+ str: String;
+include "bufio.m";
+include "zip.m";
+ zip: Zip;
+ Fhdr, CDFhdr, Endofcdir: import zip;
+
+Getzip: module {+ init: fn(nil: ref Draw->Context, args: list of string);
+};
+
+dflag: int;
+kflag: int;
+vflag: int;
+
+init(nil: ref Draw->Context, args: list of string)
+{+ sys = load Sys Sys->PATH;
+ arg := load Arg Arg->PATH;
+ str = load String String->PATH;
+ zip = load Zip Zip->PATH;
+ zip->init();
+
+ # for zip library
+ sys->pctl(Sys->FORKNS, nil);
+ if(sys->bind("#s", "/chan", Sys->MREPL) < 0)+ fail(sprint("bind /chan: %r"));+
+ arg->init(args);
+ arg->setusage(arg->progname()+" [-dkv] zipfile [path ...]");
+ while((c := arg->opt()) != 0)
+ case c {+ 'd' => zip->dflag = dflag++;
+ 'k' => kflag++;
+ 'v' => vflag++;
+ * => arg->usage();
+ }
+ args = arg->argv();
+ if(len args == 0)
+ arg->usage();
+
+ fd := sys->open(hd args, Sys->OREAD);
+ if(fd == nil)
+ fail(sprint("open: %r"));+
+ (nil, fhdrs, err) := zip->open(fd);
+ if(err != nil)
+ fail("parsing zip: "+err);+
+ all := tl args == nil;
+ paths: array of string;
+ if(!all) {+ i := 0;
+ paths = array[len args-1] of string;
+ for(l := tl args; l != nil; l = tl l)
+ paths[i++] = zip->sanitizepath(hd l);
+ }
+File:
+ for(i := 0; i < len fhdrs && (all || len paths > 0 && paths[0] != nil); i++) {+ if(!all && !match(paths, fhdrs[i].filename))
+ continue;
+
+ f := "./"+fhdrs[i].filename;
+ if(f[len f-1] == '/') {+ merr := mkdirs(f[2:]);
+ if(merr != nil) {+ warn(merr);
+ continue;
+ }
+ if(vflag)
+ sys->print("%q\n", fhdrs[i].filename);+ continue;
+ } else
+ mkdirs(str->splitstrr(f[2:], "/").t0);
+
+ (zfd, nil, zerr) := zip->openfile(fd, fhdrs[i]);
+ if(zerr != nil) {+ warn(sprint("open %q in zip file: %s", f, zerr));+ continue;
+ }
+
+ flags := Sys->OWRITE;
+ if(kflag)
+ flags = Sys->OEXCL;
+ ofd := sys->create(f, flags, 8r666);
+ if(ofd == nil) {+ warn(sprint("create %q: %r", f));+ continue;
+ }
+
+ buf := array[Sys->ATOMICIO] of byte;
+ for(;;) {+ n := sys->read(zfd, buf, len buf);
+ if(n < 0) {+ warn(sprint("reading from %q from zip: %r", f));+ continue File;
+ }
+ if(n == 0)
+ break;
+ if(sys->write(ofd, buf, n) != n)
+ warn(sprint("writing %q: %r", f));+ }
+
+ if(vflag)
+ sys->print("%q\n", fhdrs[i].filename);+ }
+
+ for(i = 0; !all && i < len paths && paths[i] != nil; i++)
+ warn(sprint("path %q not in archive", paths[i++]));+}
+
+mkdirs(s: string): string
+{+ p := "";
+ lasterr := 0;
+ for(l := sys->tokenize(s, "/").t1; l != nil; l = tl l) {+ if(p != nil)
+ p[len p] = '/';
+ p += hd l;
+ lasterr = sys->create(p, Sys->OEXCL|Sys->OREAD, Sys->DMDIR|8r777) == nil;
+ }
+ if(lasterr)
+ return sprint("create ./%q: %r", p);+ return nil;
+}
+
+match(paths: array of string, s: string): int
+{+ for(i := 0; i < len paths; i++)
+ if(paths[i] == nil) {+ return 0;
+ } else if(paths[i] == s) {+ paths[i:] = paths[i+1:];
+ paths[len paths-1] = nil;
+ return 1;
+ }
+ return 0;
+}
+
+warn(s: string)
+{+ sys->fprint(sys->fildes(2), "%s\n", s);
+}
+
+fail(s: string)
+{+ warn(s);
+ raise "fail:"+s;
+}
--- /dev/null
+++ b/appl/cmd/zip/lszip.b
@@ -1,0 +1,53 @@
+implement Lszip;
+
+include "sys.m";
+ sys: Sys;
+ sprint: import sys;
+include "draw.m";
+include "arg.m";
+include "bufio.m";
+include "zip.m";
+ zip: Zip;
+ Fhdr, CDFhdr, Endofcdir: import zip;
+
+Lszip: module {+ init: fn(nil: ref Draw->Context, args: list of string);
+};
+
+dflag: int;
+
+init(nil: ref Draw->Context, args: list of string)
+{+ sys = load Sys Sys->PATH;
+ arg := load Arg Arg->PATH;
+ zip = load Zip Zip->PATH;
+ zip->init();
+
+ arg->init(args);
+ arg->setusage(arg->progname()+" [-d] file");
+ while((c := arg->opt()) != 0)
+ case c {+ 'd' => zip->dflag = dflag++;
+ * => arg->usage();
+ }
+ args = arg->argv();
+ if(len args != 1)
+ arg->usage();
+
+ fd := sys->open(hd args, Sys->OREAD);
+ if(fd == nil)
+ fail(sprint("open: %r"));+
+ (nil, fhdrs, err) := zip->open(fd);
+ if(err != nil)
+ fail("parsing zip: "+err);+
+ for(i := 0; i < len fhdrs; i++)
+ sys->print("%q\n", fhdrs[i].filename);+}
+
+fail(s: string)
+{+ sys->fprint(sys->fildes(2), "%s\n", s);
+ raise "fail:"+s;
+}
--- /dev/null
+++ b/appl/cmd/zip/mkfile
@@ -1,0 +1,28 @@
+<../../../mkconfig
+
+TARG=\
+ lszip.dis\
+ getzip.dis\
+ putzip.dis\
+ zipfs.dis\
+ zipstream.dis\
+
+SYSMODULES=\
+ arg.m\
+ bufio.m\
+ convcs.m\
+ daytime.m\
+ draw.m\
+ encoding.m\
+ filter.m\
+ lists.m\
+ string.m\
+ styx.m\
+ styxservers.m\
+ sys.m\
+ tables.m\
+ zip.m\
+
+DISBIN=$ROOT/dis/zip
+
+<$ROOT/mkfiles/mkdis
--- /dev/null
+++ b/appl/cmd/zip/putzip.b
@@ -1,0 +1,262 @@
+implement Putzip;
+
+include "sys.m";
+ sys: Sys;
+ sprint: import sys;
+include "draw.m";
+include "arg.m";
+include "bufio.m";
+ bufio: Bufio;
+ Iobuf: import bufio;
+include "string.m";
+ str: String;
+include "lists.m";
+ lists: Lists;
+include "filter.m";
+ deflate: Filter;
+include "zip.m";
+ zip: Zip;
+ Fhdr, CDFhdr, Endofcdir: import zip;
+
+Putzip: module {+ init: fn(nil: ref Draw->Context, args: list of string);
+};
+
+dflag: int;
+vflag: int;
+pflag: int;
+
+zb: ref Iobuf;
+fileheaders: list of ref (big, ref Fhdr);
+
+init(nil: ref Draw->Context, args: list of string)
+{+ sys = load Sys Sys->PATH;
+ arg := load Arg Arg->PATH;
+ bufio = load Bufio Bufio->PATH;
+ str = load String String->PATH;
+ lists = load Lists Lists->PATH;
+ deflate = load Filter Filter->DEFLATEPATH;
+ deflate->init();
+ zip = load Zip Zip->PATH;
+ zip->init();
+
+ arg->init(args);
+ arg->setusage(arg->progname()+" [-dvp] zipfile [path ...]");
+ while((c := arg->opt()) != 0)
+ case c {+ 'd' => zip->dflag = dflag++;
+ 'v' => vflag++;
+ 'p' => pflag++;
+ * => arg->usage();
+ }
+ args = arg->argv();
+ if(len args == 0)
+ arg->usage();
+
+ zfd := sys->create(hd args, Sys->OWRITE|Sys->OEXCL, 8r666);
+ if(zfd == nil)
+ fail(sprint("create %q: %r", hd args));+ zb = bufio->fopen(zfd, Sys->OWRITE);
+ if(zb == nil)
+ fail(sprint("fopen: %r"));+
+ args = tl args;
+ if(args == nil) {+ b := bufio->fopen(sys->fildes(0), Bufio->OREAD);
+ if(b == nil)
+ fail(sprint("fopen: %r"));+ for(;;) {+ s := b.gets('\n');+ if(s == nil)
+ break;
+ if(s != nil && s[len s-1] == '\n')
+ s = s[:len s-1];
+ put(s, 0);
+ }
+ } else {+ for(; args != nil; args = tl args)
+ put(hd args, 1);
+ }
+
+ eocd := ref Endofcdir (0, 0, len fileheaders, len fileheaders, big 0, big 0, nil);
+ eocd.cdiroffset = zb.offset();
+ for(l := lists->reverse(fileheaders); l != nil; l = tl l) {+ (foff, f) := *hd l;
+ cdf := CDFhdr.mk(f, foff);
+ buf := cdf.pack();
+ if(zb.write(buf, len buf) != len buf)
+ fail(sprint("writing central directory file header: %r"));+ eocd.cdirsize += big len buf;
+ }
+
+ ebuf := eocd.pack();
+ if(zb.write(ebuf, len ebuf) != len ebuf || zb.flush() == Bufio->ERROR)
+ fail(sprint("writing end of central directory: %r"));+}
+
+put(s: string, recur: int)
+{+ if(s == nil)
+ warn("refusing to add empty filename");+
+ fd := sys->open(s, Sys->OREAD);
+ if(fd == nil)
+ return warn(sprint("open %q: %r, skipping", s));+ (ok, dir) := sys->fstat(fd);
+ if(ok < 0)
+ return warn(sprint("fstat %q: %r, skipping", s));+
+ if(dir.mode & Sys->DMDIR)
+ putdir(s, fd, dir, recur);
+ else
+ putfile(s, fd, dir);
+}
+
+mkfhdr(mtime: int, s: string): ref Fhdr
+{+ f := ref Fhdr;
+ f.versneeded = 20;
+ f.flags = zip->Futf8;
+ f.comprmethod = zip->Mdeflate;
+ if(pflag)
+ f.comprmethod = zip->Mplain;
+ f.mtime = mtime;
+ f.filename = zip->sanitizepath(s);
+ f.comprsize = big 0;
+ f.uncomprsize = big 0;
+ f.crc32 = big 0;
+ return f;
+}
+
+putdir(s: string, fd: ref Sys->FD, dir: Sys->Dir, recur: int)
+{+ if(s[len s-1] != '/')
+ s[len s] = '/';
+ f := mkfhdr(dir.mtime, s);
+ foff := zb.offset();
+ fbuf := f.pack();
+ if(zb.write(fbuf, len fbuf) != len fbuf)
+ fail(sprint("write: %r"));+ fileheaders = ref (foff, f)::fileheaders;
+ if(vflag)
+ sys->print("%s\n", s);+
+ if(!recur)
+ return;
+
+ for(;;) {+ (n, dirs) := sys->dirread(fd);
+ if(n < 0)
+ return warn(sprint("listing %q: %r", s));+ if(n == 0)
+ break;
+ for(i := 0; i < len dirs; i++)
+ put(s+dirs[i].name, recur);
+ }
+}
+
+putfile(s: string, fd: ref Sys->FD, dir: Sys->Dir)
+{+ f := mkfhdr(dir.mtime, s);
+ if(vflag)
+ sys->print("%s\n", s);+
+ foff := zb.offset();
+
+ # write partially filled header, prevents fs holes
+ fbuf := f.pack();
+ if(zb.write(fbuf, len fbuf) != len fbuf)
+ fail(sprint("write: %r"));+
+ if(f.comprmethod == zip->Mplain)
+ putplain(fd, f);
+ else
+ putdeflate(fd, f, s);
+
+ # rewrite file header, now complete. restore offset afterwards
+ off := zb.offset();
+ fbuf = f.pack();
+ if(zb.seek(foff, Bufio->SEEKSTART) < big 0)
+ fail(sprint("seek to file header: %r"));+ if(zb.write(fbuf, len fbuf) != len fbuf)
+ fail(sprint("write %q file header: %r", s));+ if(zb.seek(off, Bufio->SEEKSTART) < big 0)
+ fail(sprint("seek to past compressed contents: %r"));+
+ fileheaders = ref (foff, f)::fileheaders;
+}
+
+putplain(fd: ref Sys->FD, f: ref Fhdr)
+{+ crc := ~0;
+
+ buf := array[sys->ATOMICIO] of byte;
+ for(;;) {+ n := sys->read(fd, buf, len buf);
+ if(n == 0)
+ break;
+ if(n < 0)
+ fail(sprint("read: %r"));+ if(zb.write(buf, n) != n)
+ fail(sprint("write: %r"));+ crc = zip->crc32(crc, buf[:n]);
+ f.uncomprsize += big n;
+ }
+ f.comprsize = f.uncomprsize;
+ f.crc32 = big ~crc;
+}
+
+putdeflate(fd: ref Sys->FD, f: ref Fhdr, s: string)
+{+ rqc := deflate->start("");+ pick r := <-rqc {+ Start => ;
+ * => fail(sprint("bad first filter msg"));+ }
+
+ crc := ~0;
+Filter:
+ for(;;) pick rq := <-rqc {+ Fill =>
+ n := sys->read(fd, rq.buf, len rq.buf);
+ if(n >= 0)
+ crc = zip->crc32(crc, rq.buf[:n]);
+ rq.reply <-= n;
+ if(n < 0)
+ fail(sprint("reading %q: %r", s));+ f.uncomprsize += big n;
+ Result =>
+ if(zb.write(rq.buf, len rq.buf) != len rq.buf)
+ fail(sprint("writing %q compressed: %r", s));+ f.comprsize += big len rq.buf;
+ rq.reply <-= 0;
+ Finished =>
+ if(len rq.buf != 0)
+ fail(sprint("deflate leftover bytes..."));+ break Filter;
+ Info =>
+ say("deflate: "+rq.msg);+ Error =>
+ fail("deflate error: "+rq.e);+ }
+
+ f.crc32 = big ~crc;
+}
+
+warn(s: string)
+{+ sys->fprint(sys->fildes(2), "%s\n", s);
+}
+
+say(s: string)
+{+ if(dflag)
+ warn(s);
+}
+
+fail(s: string)
+{+ warn(s);
+ raise "fail:"+s;
+}
--- /dev/null
+++ b/appl/cmd/zip/zipfs.b
@@ -1,0 +1,321 @@
+# zipfs design
+#
+# at startup we read the entire central directory,
+# set up a styxservers nametree and serve from that.
+# bit 62 of the qid denotes the directory bit.
+#
+# if a file is not compressed and -p was set, we read directly through
+# the zipfile's fd (no crc protection).
+# otherwise, on first read of a file, we open a fd for that file.
+# sequential or higher-than-current offset reads reuse the fd.
+# random reads to before the current offset reopen the file and
+# seek to the desired location by reading & discarding data.
+
+implement Zipfs;
+
+include "sys.m";
+ sys: Sys;
+ sprint: import sys;
+include "draw.m";
+include "arg.m";
+include "bufio.m";
+ bufio: Bufio;
+ Iobuf: import bufio;
+include "string.m";
+ str: String;
+include "daytime.m";
+ daytime: Daytime;
+include "styx.m";
+ styx: Styx;
+ Tmsg, Rmsg: import Styx;
+include "styxservers.m";
+ styxservers: Styxservers;
+ Styxserver, Fid, Navigator, Navop: import styxservers;
+ nametree: Nametree;
+ Tree: import nametree;
+include "tables.m";
+ tables: Tables;
+ Table, Strhash: import tables;
+include "zip.m";
+ zip: Zip;
+ Fhdr, CDFhdr, Endofcdir: import zip;
+
+Zipfs: module {+ init: fn(nil: ref Draw->Context, args: list of string);
+};
+
+
+# directories for styx nametree
+Zdir: adt {+ name: string;
+ qid: big;
+ pqid: big;
+};
+
+# opened file from zip archive
+Zfile: adt {+ fid: int;
+ fd: ref Sys->FD;
+ off: big;
+ cdf: ref CDFhdr;
+ f: ref Fhdr;
+};
+
+Qiddir: con big 1<<62;
+
+zipfd: ref Sys->FD;
+srv: ref Styxserver;
+files: array of ref Fhdr;
+zdirs: ref Strhash[ref Zdir]; # indexed by full path without trailing slash
+rootzdir: ref Zdir;
+dirgen: int;
+
+cdirfhdrs: array of ref CDFhdr;
+
+zfiles: ref Table[ref Zfile]; # indexed by fid
+
+Dflag: int;
+dflag: int;
+pflag: int;
+now: int;
+
+init(nil: ref Draw->Context, args: list of string)
+{+ sys = load Sys Sys->PATH;
+ arg := load Arg Arg->PATH;
+ bufio = load Bufio Bufio->PATH;
+ str = load String String->PATH;
+ daytime = load Daytime Daytime->PATH;
+ styx = load Styx Styx->PATH;
+ styx->init();
+ styxservers = load Styxservers Styxservers->PATH;
+ styxservers->init(styx);
+ nametree = load Nametree Nametree->PATH;
+ nametree->init();
+ tables = load Tables Tables->PATH;
+ zip = load Zip Zip->PATH;
+ zip->init();
+
+ # for zip library
+ sys->pctl(Sys->NEWPGRP|Sys->FORKNS, nil);
+ if(sys->bind("#s", "/chan", Sys->MREPL) < 0)+ fail(sprint("bind /chan: %r"));+
+ arg->init(args);
+ arg->setusage(arg->progname()+" [-Ddp] zipfile");
+ while((c := arg->opt()) != 0)
+ case c {+ 'D' => styxservers->traceset(Dflag++);
+ 'd' => zip->dflag = dflag++;
+ 'p' => pflag++;
+ * => arg->usage();
+ }
+ args = arg->argv();
+ if(len args != 1)
+ arg->usage();
+
+ zdirs = zdirs.new(101, nil);
+
+ zipfd = sys->open(hd args, Sys->OREAD);
+ if(zipfd == nil)
+ fail(sprint("open: %r"));+
+ err: string;
+ (nil, cdirfhdrs, err) = zip->open(zipfd);
+ if(err != nil)
+ fail("parsing zip: "+err);+
+ now = daytime->now();
+ (tree, treeop) := nametree->start();
+ rzd := rootzdir = ref Zdir (".", big 0|Qiddir, big 0|Qiddir);+ tree.create(rzd.pqid, dir(rzd.name, rzd.qid, now, big 0));
+
+ for(i := 0; i < len cdirfhdrs; i++) {+ f := cdirfhdrs[i];
+ fs := f.filename;
+ (path, name) := str->splitstrr(fs, "/");
+ zd := ensuredir(tree, path);
+ if(fs == nil || fs[len fs-1] == '/')
+ continue;
+ qid := big (i+1);
+ tree.create(zd.qid, dir(name, qid, f.mtime, f.uncomprsize));
+ }
+ zdirs = nil;
+
+ zfiles = zfiles.new(31, nil);
+
+ msgc: chan of ref Tmsg;
+ (msgc, srv) = Styxserver.new(sys->fildes(0), Navigator.new(treeop), big 0|Qiddir);
+ spawn styxsrv(msgc);
+}
+
+# inefficient
+# called with empty string (denoting the root directory), or path with trailing slash
+ensuredir(tree: ref Tree, s: string): ref Zdir
+{+ if(s == nil)
+ return rootzdir;
+ s = s[:len s-1];
+
+ zd := zdirs.find(s);
+ if(zd != nil)
+ return zd;
+
+ (ppath, name) := str->splitstrr(s, "/");
+ pzd := ensuredir(tree, ppath);
+ zd = ref Zdir (name, big ++dirgen|Qiddir, pzd.qid);
+ tree.create(pzd.qid, dir(zd.name, zd.qid, now, big 0));
+ zdirs.add(s, zd);
+ return zd;
+}
+
+dir(name: string, qid: big, mtime: int, length: big): Sys->Dir
+{+ d := sys->zerodir;
+ d.name = name;
+ d.uid = "zip";
+ d.gid = "zip";
+ d.qid.path = qid;
+ if((qid & Qiddir) != big 0) {+ d.qid.qtype = Sys->QTDIR;
+ d.mode = Sys->DMDIR|8r555;
+ } else {+ d.qid.qtype = Sys->QTFILE;
+ d.mode = 8r444;
+ }
+ d.mtime = d.atime = mtime;
+ d.length = length;
+ return d;
+}
+
+styxsrv(msgc: chan of ref Tmsg)
+{+done:
+ for(;;) alt {+ mm := <-msgc =>
+ if(mm == nil)
+ break done;
+ pick m := mm {+ Readerror =>
+ warn("read error: "+m.error);+ break done;
+ }
+ dostyx(mm);
+ }
+ killgrp(sys->pctl(0, nil));
+}
+
+openzfile(fid: int, qid: big): (ref Zfile, string)
+{+ cdf := cdirfhdrs[int qid-1];
+ zf := ref Zfile (fid, nil, big 0, cdf, nil);
+ err: string;
+ if(pflag && cdf.comprmethod == zip->Mplain)
+ (zf.f, err) = zip->readfhdr(zipfd, cdf);
+ else
+ (zf.fd, nil, err) = zip->openfile(zipfd, cdf);
+ if(err != nil)
+ return (nil, err);
+ if(Dflag) warn("zipfs: "+cdf.filename);+ zfiles.add(fid, zf);
+ return (zf, nil);
+}
+
+zfileseek(zf: ref Zfile, off: big): string
+{+ if(zf.off == off)
+ return nil;
+
+ if(off < zf.off) {+ (fd, nil, err) := zip->openfile(zipfd, zf.cdf);
+ if(err != nil)
+ return err;
+ zf.fd = fd;
+ zf.off = big 0;
+ }
+
+ n := int (off-zf.off);
+ buf := array[Sys->ATOMICIO] of byte;
+ while(n > 0) {+ nn := sys->read(zf.fd, buf, len buf);
+ if(nn < 0)
+ return sprint("seeking to requested offset: %r");+ if(nn == 0)
+ break; # not there yet, but subsequent reads on zf.fd will/should return eof too
+ zf.off += big nn;
+ n -= nn;
+ }
+
+ return nil;
+}
+
+dostyx(mm: ref Tmsg)
+{+ pick m := mm {+ Clunk or
+ Remove =>
+ f := srv.getfid(m.fid);
+ if(f != nil && f.isopen && (f.path & Qiddir) == big 0)
+ zfiles.del(m.fid);
+ srv.default(m);
+
+ Read =>
+ f := srv.getfid(m.fid);
+ if(f.qtype & Sys->QTDIR)
+ return srv.default(m);
+
+ zf := zfiles.find(m.fid);
+ if(zf == nil) {+ err: string;
+ (zf, err) = openzfile(m.fid, f.path);
+ if(err != nil)
+ return replyerror(m, err);
+ }
+
+ if(zf.f == nil) {+ err := zfileseek(zf, m.offset);
+ if(err != nil)
+ return replyerror(m, err);
+ n := sys->read(zf.fd, buf := array[m.count] of byte, len buf);
+ if(n < 0)
+ return replyerror(m, sprint("%r"));+ zf.off += big n;
+ srv.reply(ref Rmsg.Read(m.tag, buf[:n]));
+ } else {+ n := zip->pread(zipfd, zf.f, buf := array[m.count] of byte, len buf, m.offset);
+ if(n < 0)
+ return replyerror(m, sprint("%r"));+ srv.reply(ref Rmsg.Read(m.tag, buf[:n]));
+ }
+
+ * =>
+ srv.default(mm);
+ }
+}
+
+replyerror(m: ref Tmsg, s: string)
+{+ srv.reply(ref Rmsg.Error(m.tag, s));
+}
+
+killgrp(pid: int)
+{+ sys->fprint(sys->open(sprint("/prog/%d/ctl", pid), Sys->OWRITE), "killgrp");+}
+
+warn(s: string)
+{+ sys->fprint(sys->fildes(2), "%s\n", s);
+}
+
+say(s: string)
+{+ if(dflag)
+ warn(s);
+}
+
+fail(s: string)
+{+ warn(s);
+ raise "fail:"+s;
+}
--- /dev/null
+++ b/appl/cmd/zip/zipstream.b
@@ -1,0 +1,209 @@
+# rewrite zip file, putting central directory at the front of the file.
+
+implement Zipstream;
+
+include "sys.m";
+ sys: Sys;
+ sprint: import sys;
+include "draw.m";
+include "arg.m";
+include "bufio.m";
+ bufio: Bufio;
+ Iobuf: import bufio;
+include "string.m";
+ str: String;
+include "filter.m";
+ deflate: Filter;
+include "zip.m";
+ zip: Zip;
+ Fhdr, CDFhdr, Endofcdir: import zip;
+
+Zipstream: module {+ init: fn(nil: ref Draw->Context, args: list of string);
+};
+
+dflag: int;
+oflag: int;
+
+init(nil: ref Draw->Context, args: list of string)
+{+ sys = load Sys Sys->PATH;
+ arg := load Arg Arg->PATH;
+ bufio = load Bufio Bufio->PATH;
+ str = load String String->PATH;
+ deflate = load Filter Filter->DEFLATEPATH;
+ deflate->init();
+ zip = load Zip Zip->PATH;
+ zip->init();
+
+ arg->init(args);
+ arg->setusage(arg->progname()+" [-d] [-o] zipfile");
+ while((c := arg->opt()) != 0)
+ case c {+ 'd' => zip->dflag = dflag++;
+ 'o' => oflag++;
+ }
+ args = arg->argv();
+ if(len args != 1)
+ arg->usage();
+
+ fd := sys->open(hd args, Sys->OREAD);
+ if(fd == nil)
+ fail(sprint("open: %r"));+
+ (eocd, cdfhdrs, err) := zip->open(fd);
+ if(err != nil)
+ fail("parsing zip: "+err);+
+ if(oflag) {+ files := readfiles();
+ n := array[len cdfhdrs] of ref CDFhdr;
+ for(i := 0; i < len files; i++) {+ j := find(cdfhdrs, files[i]);
+ if(j < 0)
+ fail(sprint("%#q not in zip file or specified twice", files[i]));+ n[i] = cdfhdrs[j];
+ cdfhdrs[j] = nil;
+ }
+ for(j := 0; j < len cdfhdrs; j++)
+ if(cdfhdrs[j] != nil)
+ n[i++] = cdfhdrs[j];
+ cdfhdrs = n;
+ }
+
+ zb := bufio->fopen(sys->fildes(1), bufio->OWRITE);
+ if(zb == nil)
+ fail(sprint("fopen: %r"));+
+ # calculate size of cdfhdrs
+ o := big 0;
+ for(i := 0; i < len cdfhdrs; i++)
+ o += big len cdfhdrs[i].pack();
+ cdsz := o;
+
+ eocd.cdirsize = cdsz;
+ eocd.cdiroffset = big 0;
+ o += big len eocd.pack();
+
+ # fix the offsets in the cdfhdrs & fhdrs, store original offsets
+ origoff := array[len cdfhdrs] of big;
+ fhdrs := array[len cdfhdrs] of ref Fhdr;
+ for(i = 0; i < len fhdrs; i++) {+ cdf := cdfhdrs[i];
+ f: ref Fhdr;
+ (f, err) = Fhdr.read(fd, cdf.reloffset);
+ if(err != nil)
+ fail("reading local file header: "+err);+ origoff[i] = f.dataoff;
+ cdf.reloffset = o;
+ fhdrs[i] = Fhdr.mk(cdf);
+ o += big len fhdrs[i].pack()+fhdrs[i].comprsize;
+ }
+
+ # write the cdfhdrs
+ for(i = 0; i < len cdfhdrs; i++) {+ if(zb.write(buf := cdfhdrs[i].pack(), len buf) != len buf)
+ fail(sprint("write: %r"));+ }
+ # a copy of the end of central directory structure (to keep other zip code happy)
+ if(zb.write(buf := eocd.pack(), len buf) != len buf)
+ fail(sprint("write: %r"));+
+ # write the fhdrs & file contents
+ for(i = 0; i < len fhdrs; i++) {+ if(zb.write(buf = fhdrs[i].pack(), len buf) != len buf || copyrange(fd, origoff[i], fhdrs[i].comprsize, zb) < 0)
+ fail(sprint("write: %r"));+ }
+
+ if(zb.offset() != o)
+ fail(sprint("inconsitent offset after rewriting central directory and files, expected offset %bd, saw %bd", o, zb.offset()));+
+ # write the eocd, this one is typically found by code parsing the zip file
+ if(zb.write(buf = eocd.pack(), len buf) != len buf || zb.flush() == bufio->ERROR)
+ fail(sprint("write: %r"));+}
+
+readfiles(): array of string
+{+ b := bufio->fopen(sys->fildes(0), bufio->OREAD);
+ if(b == nil)
+ fail(sprint("fopen: %r"));+ l: list of string;
+ for(;;) {+ s := b.gets('\n');+ if(s == nil)
+ break;
+ if(s[len s-1] == '\n')
+ s = s[:len s-1];
+ l = s::l;
+ }
+ f := array[len l] of string;
+ i := len f-1;
+ for(; l != nil; l = tl l)
+ f[i--] = hd l;
+ return f;
+}
+
+find(h: array of ref CDFhdr, s: string): int
+{+ for(i := 0; i < len h; i++)
+ if(h[i] != nil && h[i].filename == s)
+ return i;
+ return -1;
+}
+
+copyrange(fd: ref Sys->FD, off: big, n: big, zb: ref Iobuf): int
+{+ buf := array[sys->ATOMICIO] of byte;
+ while(n > big 0) {+ if(n > big len buf)
+ nn := len buf;
+ else
+ nn = int n;
+ r := preadn(fd, buf, nn, off);
+ if(r < 0)
+ return -1;
+ else if(r != nn) {+ sys->werrstr("short read");+ return -1;
+ }
+ if(zb.write(buf, r) != r)
+ return -1;
+ off += big r;
+ n -= big r;
+ }
+ return 0;
+}
+
+preadn(fd: ref Sys->FD, buf: array of byte, n: int, off: big): int
+{+ org := n;
+ while(n > 0) {+ nn := sys->pread(fd, buf, n, off);
+ if(nn < 0)
+ return nn;
+ if(nn == 0)
+ break;
+ n -= nn;
+ off += big nn;
+ buf = buf[nn:];
+ }
+ return org-n;
+}
+
+warn(s: string)
+{+ sys->fprint(sys->fildes(2), "%s\n", s);
+}
+
+say(s: string)
+{+ if(dflag)
+ warn(s);
+}
+
+fail(s: string)
+{+ warn(s);
+ raise "fail:"+s;
+}
--- a/appl/lib/mkfile
+++ b/appl/lib/mkfile
@@ -155,6 +155,7 @@
workdir.dis\
writegif.dis\
xml.dis\
+ zip.dis\
MODULES=
--- /dev/null
+++ b/appl/lib/zip.b
@@ -1,0 +1,917 @@
+implement Zip;
+
+include "sys.m";
+ sys: Sys;
+ sprint: import sys;
+include "draw.m";
+include "arg.m";
+include "bufio.m";
+ bufio: Bufio;
+ Iobuf: import bufio;
+include "daytime.m";
+ dt: Daytime;
+include "encoding.m";
+ base16: Encoding;
+include "filter.m";
+ inflate: Filter;
+include "convcs.m";
+ convcs: Convcs;
+ cp437: Btos;
+include "zip.m";
+
+comprmethods := array[] of {+ "uncompressed",
+ "shrunk",
+ "reduce, factor 1",
+ "reduce, factor 2",
+ "reduce, factor 3",
+ "reduce, factor 4",
+ "implode",
+ "tokenize",
+ "deflate",
+ "deflate64",
+ "pkware implode",
+ nil,
+ "bzip2",
+ nil,
+ "lzma",
+Mibmterse =>
+ "ibm terse (new)",
+Mlz77z =>
+ "ibm lz77 z",
+Mwavpack =>
+ "wavpack",
+Mppmdi1 =>
+ "PPMd version I, rev 1",
+};
+
+Crc32poly: con int 16redb88320; # reversed standard crc-32
+crc32tab: array of int;
+
+
+init(): string
+{+ sys = load Sys Sys->PATH;
+ bufio = load Bufio Bufio->PATH;
+ base16 = load Encoding Encoding->BASE16PATH;
+ dt = load Daytime Daytime->PATH;
+ inflate = load Filter Filter->INFLATEPATH;
+ inflate->init();
+ convcs = load Convcs Convcs->PATH;
+ err := convcs->init(nil);
+ if(err == nil)
+ (cp437, err) = convcs->getbtos("cp437");+ if(err != nil)
+ err = "convcs: "+err;
+
+ crc32tab = mkcrctab(Crc32poly);
+
+ return err;
+}
+
+
+sanitizepath(s: string): string
+{+ endslash := s != nil && s[len s-1] == '/';
+ r: list of string;
+ for(l := sys->tokenize(s, "/").t1; l != nil; l = tl l)
+ case hd l {+ "." => ;
+ ".." => if(r != nil) r = tl r;
+ * => r = hd l::r;
+ }
+ rs := "";
+ for(r = rev(r); r != nil; r = tl r)
+ rs += "/"+hd r;
+ if(rs != nil)
+ rs = rs[1:];
+ if(endslash)
+ rs[len rs] = '/';
+ return rs;
+}
+
+Extra.parse(d: array of byte): (ref Extra, string)
+{+ e := ref Extra;
+ {+ o := 0;
+ while(o < len d) {+ id: int;
+ buf: array of byte;
+ (id, o) = g16(d, o);
+ (buf, o) = gstr(d, o);
+ e.l = ref (id, buf)::e.l;
+ }
+ } exception {+ "get:*" =>
+ return (nil, sprint("bad extra"));+ }
+ e.l = rev(e.l);
+ return (e, nil);
+}
+
+Extra.pack(e: self ref Extra): array of byte
+{+ n := 0;
+ if(e != nil)
+ l := e.l;
+ for(t := l; t != nil; t = tl t)
+ n += 2+2+len (hd t).t1;
+ buf := array[n] of byte;
+ o := 0;
+ for(; l != nil; l = tl l) {+ (id, dat) := *hd l;
+ o = p16(buf, o, id);
+ o = p16(buf, o, len dat);
+ o = pbuf(buf, o, dat);
+ }
+ return buf;
+}
+
+Extra.text(e: self ref Extra): string
+{+ s: string;
+ for(l := e.l; l != nil; l = tl l)
+ s += sprint(", 0x%04ux=%s", (hd l).t0, base16->enc((hd l).t1));+ if(s != nil)
+ s = s[2:];
+ return "Extra("+s+")";+}
+
+
+Fhdr.mk(x: ref CDFhdr): ref Fhdr
+{+ return ref Fhdr (
+ x.versneeded,
+ x.flags,
+ x.comprmethod,
+ x.filemtime,
+ x.filemdate,
+ x.mtime,
+ x.crc32,
+ x.comprsize,
+ x.uncomprsize,
+ x.filename,
+ ref *x.extra,
+ big 0);
+}
+
+fhdrsig := array[] of {byte 'P', byte 'K', byte 3, byte 4};+Fhdr.parse(buf: array of byte, off: big): (ref Fhdr, string)
+{+ if(len buf < 4 || bufcmp(buf[:4], fhdrsig) != 0)
+ return (nil, sprint("not a local file header"));+ o := 4;
+ f := ref Fhdr;
+ {+ (f.versneeded, o) = g16(buf, o);
+ (f.flags, o) = g16(buf, o);
+ (f.comprmethod, o) = g16(buf, o);
+ (f.filemtime, o) = g16(buf, o);
+ (f.filemdate, o) = g16(buf, o);
+ f.mtime = mtimedos2unix(f.filemtime, f.filemdate);
+ (f.crc32, o) = g32(buf, o);
+ (f.comprsize, o) = g32(buf, o);
+ (f.uncomprsize, o) = g32(buf, o);
+ flen, extralen: int;
+ (flen, o) = g16(buf, o);
+ (extralen, o) = g16(buf, o);
+ (f.filename, o) = gbufstr(f.flags&Futf8, flen, buf, o);
+ f.filename = sanitizepath(f.filename);
+ extra: array of byte;
+ (extra, o) = gbuf(extralen, buf, o);
+ err: string;
+ (f.extra, err) = Extra.parse(extra);
+ if(err != nil)
+ return (nil, "bad extra for local file header: "+err);
+ f.dataoff = off+big o;
+ } exception {+ "get:*" =>
+ return (nil, sprint("short buffer for local file header (o %d, len %d)", o, len buf));+ }
+ return (f, nil);
+}
+
+Fhdr.read(fd: ref Sys->FD, off: big): (ref Fhdr, string)
+{+ minwidth: con 4+5*2+3*4+2*2;
+ n := preadn(fd, buf0 := array[minwidth] of byte, len buf0, off);
+ if(n < 0)
+ return (nil, sprint("read: %r"));+ if(n != len buf0)
+ return (nil, "short read");
+ flen := g16(buf0, len buf0-4).t0;
+ extralen := g16(buf0, len buf0-2).t0;
+ buf1 := array[flen+extralen] of byte;
+ if(len buf1 > 0) {+ n = preadn(fd, buf1, len buf1, off+big minwidth);
+ if(n < 0)
+ return (nil, sprint("read: %r"));+ if(n != len buf1)
+ return (nil, "short read");
+ }
+ buf := array[len buf0+len buf1] of byte;
+ buf[:] = buf0;
+ buf[len buf0:] = buf1;
+ return Fhdr.parse(buf, off);
+}
+
+Fhdr.pack(f: self ref Fhdr): array of byte
+{+ filename := array of byte f.filename;
+ ebuf := f.extra.pack();
+ buf := array[4+5*2+3*4+2+len filename+2+len ebuf] of byte;
+
+ (f.filemtime, f.filemdate) = mtimeunix2dos(f.mtime);
+
+ o := 0;
+ o = pbuf(buf, o, fhdrsig);
+ o = p16(buf, o, f.versneeded);
+ o = p16(buf, o, f.flags);
+ o = p16(buf, o, f.comprmethod);
+ o = p16(buf, o, f.filemtime);
+ o = p16(buf, o, f.filemdate);
+ o = p32(buf, o, f.crc32);
+ o = p32(buf, o, f.comprsize);
+ o = p32(buf, o, f.uncomprsize);
+ o = p16(buf, o, len filename);
+ o = p16(buf, o, len ebuf);
+ o = pbuf(buf, o, filename);
+ o = pbuf(buf, o, ebuf);
+ return buf;
+}
+
+Fhdr.text(f: self ref Fhdr): string
+{+ return sprint("Fhdr(versneeded %d (%s), flags %ux, comprmethod %d/%s, mtime %d, crc32 %bd, comprsize %bd, uncomprsize %bd, filename %q, %s, dataoff %bd)",+ f.versneeded, versstr(f.versneeded),
+ f.flags,
+ f.comprmethod, comprmethod(f.comprmethod),
+ f.mtime,
+ f.crc32,
+ f.comprsize, f.uncomprsize,
+ f.filename,
+ f.extra.text(),
+ f.dataoff);
+}
+
+
+mask(n: int): int
+{+ return (1<<n)-1;
+}
+
+mtimeunix2dos(m: int): (int, int)
+{+ tm := dt->local(m);
+ s := tm.sec | tm.min<<5 | tm.hour<<11;
+ d := tm.mday | (tm.mon+1)<<5 | (tm.year-80)<<9;
+ return (s, d);
+}
+
+zerotm: ref dt->Tm;
+mtimedos2unix(s, d: int): int
+{+ if(zerotm == nil)
+ zerotm = dt->local(dt->now());
+ tm := ref *zerotm;
+ tm.sec = (s>>0) & mask(5);
+ tm.min = (s>>5) & mask(6);
+ tm.hour = (s>>11) & mask(5);
+ tm.mday = (d>>0) & mask(5);
+ tm.mon = ((d>>5) & mask(4))-1;
+ tm.year = ((d>>9) & mask(7))+80;
+ tm.wday = 0;
+ tm.yday = 0;
+ return dt->tm2epoch(tm);
+}
+
+CDFhdr.mk(f: ref Fhdr, off: big): ref CDFhdr
+{+ return ref CDFhdr (
+ f.versneeded, # we're not claiming to be unix, because then unzip sets 0 for permissions when absent
+ f.versneeded,
+ f.flags,
+ f.comprmethod,
+ f.filemtime,
+ f.filemdate,
+ f.mtime,
+ f.crc32,
+ f.comprsize,
+ f.uncomprsize,
+ f.filename,
+ f.extra,
+ nil,
+ 0,
+ 0,
+ big 0,
+ off);
+}
+
+cdirfhdrsig := array[] of {byte 'P', byte 'K', byte 1, byte 2};+
+CDFhdr.parse(buf: array of byte): (ref CDFhdr, string)
+{+ if(len buf < 4 || bufcmp(buf[:4], cdirfhdrsig) != 0)
+ return (nil, "not a central directory file header");
+ f := ref CDFhdr;
+ o := 4;
+ {+ (f.versmadeby, o) = g16(buf, o);
+ (f.versneeded, o) = g16(buf, o);
+ (f.flags, o) = g16(buf, o);
+ if(f.flags & Fcdirencrypted)
+ return (nil, "central directory is encrypted, not supported");
+ (f.comprmethod, o) = g16(buf, o);
+ (f.filemtime, o) = g16(buf, o);
+ (f.filemdate, o) = g16(buf, o);
+ f.mtime = mtimedos2unix(f.filemtime, f.filemdate);
+ (f.crc32, o) = g32(buf, o);
+ (f.comprsize, o) = g32(buf, o);
+ (f.uncomprsize, o) = g32(buf, o);
+ flen, extralen, commentlen: int;
+ (flen, o) = g16(buf, o);
+ (extralen, o) = g16(buf, o);
+ (commentlen, o) = g16(buf, o);
+ (f.disknrstart, o) = g16(buf, o);
+ (f.intattr, o) = g16(buf, o);
+ (f.extattr, o) = g32(buf, o);
+ (f.reloffset, o) = g32(buf, o);
+ (f.filename, o) = gbufstr(f.flags&Futf8, flen, buf, o);
+ f.filename = sanitizepath(f.filename);
+ extra: array of byte;
+ (extra, o) = gbuf(extralen, buf, o);
+ err: string;
+ (f.extra, err) = Extra.parse(extra);
+ if(err != nil)
+ return (nil, sprint("bad extra for central directory file header"));+ (f.comment, o) = gbufstr(f.flags&Futf8, commentlen, buf, o);
+ if(o != len buf)
+ say(sprint("%d trailing bytes after parsing central directory file header", len buf-o));+ } exception {+ "get:*" =>
+ return (nil, sprint("short buffer for central directory file header (o %d, len %d)", o, len buf));+ }
+ return (f, nil);
+}
+
+CDFhdr.read(b: ref Bufio->Iobuf): (ref CDFhdr, string)
+{+ buf0 := array[4+6*2+3*4+5*2+2*4] of byte;
+ if(breadn(b, buf0, len buf0) != len buf0)
+ return (nil, sprint("short read on central directory file header"));+ if(bufcmp(buf0[:4], cdirfhdrsig) != 0)
+ return (nil, sprint("not signature of central directory file header"));+ lenoff := 4+6*2+3*4;
+ n := g16(buf0, lenoff).t0;
+ n += g16(buf0, lenoff+2).t0;
+ n += g16(buf0, lenoff+4).t0;
+ buf1 := array[n] of byte;
+ if(breadn(b, buf1, len buf1) != len buf1)
+ return (nil, sprint("short read on filename/extra/comment section of central directory file header"));+ buf := array[len buf0+len buf1] of byte;
+ buf[:] = buf0;
+ buf[len buf0:] = buf1;
+ return CDFhdr.parse(buf);
+}
+
+CDFhdr.pack(f: self ref CDFhdr): array of byte
+{+ filename := array of byte f.filename;
+ comment := array of byte f.comment;
+ ebuf := f.extra.pack();
+ buf := array[4+6*2+3*4+2+len filename+2+len ebuf+2+len comment+2*2+2*4] of byte;
+
+ (f.filemtime, f.filemdate) = mtimeunix2dos(f.mtime);
+
+ o := 0;
+ o = pbuf(buf, o, cdirfhdrsig);
+ o = p16(buf, o, f.versmadeby);
+ o = p16(buf, o, f.versneeded);
+ o = p16(buf, o, f.flags);
+ o = p16(buf, o, f.comprmethod);
+ o = p16(buf, o, f.filemtime);
+ o = p16(buf, o, f.filemdate);
+ o = p32(buf, o, f.crc32);
+ o = p32(buf, o, f.comprsize);
+ o = p32(buf, o, f.uncomprsize);
+ o = p16(buf, o, len filename);
+ o = p16(buf, o, len ebuf);
+ o = p16(buf, o, len comment);
+ o = p16(buf, o, f.disknrstart);
+ o = p16(buf, o, f.intattr);
+ o = p32(buf, o, f.extattr);
+ o = p32(buf, o, f.reloffset);
+ o = pbuf(buf, o, filename);
+ o = pbuf(buf, o, ebuf);
+ o = pbuf(buf, o, comment);
+ return buf;
+}
+
+CDFhdr.text(f: self ref CDFhdr): string
+{+ return sprint("CDFhdr(version: madeby %d (%s), needed %d (%s); flags %02ux, comprmethod %d/%s, mtime %d, crc32 %bux, comprsize %bd, uncomprsize %bd, file %q, %s, comment %q, disknrstart %d, intattr %02x, extattr %04bux, reloffset %bd)",+ f.versmadeby, versstr(f.versmadeby),
+ f.versneeded, versstr(f.versneeded),
+ f.flags, f.comprmethod, comprmethod(f.comprmethod),
+ f.mtime, f.crc32, f.comprsize, f.uncomprsize,
+ f.filename,
+ f.extra.text(),
+ f.comment,
+ f.disknrstart,
+ f.intattr, f.extattr, f.reloffset);
+}
+
+
+eocentraldirsig := array[] of {byte 'P', byte 'K', byte 5, byte 6};+Endofcdir.parse(buf: array of byte): (ref Endofcdir, string)
+{+ e := ref Endofcdir;
+ if(len buf < 4 || bufcmp(buf[:4], eocentraldirsig) != 0)
+ return (nil, "not end of central directory");
+ o := 4;
+ {+ (e.disknr, o) = g16(buf, o);
+ (e.diskcdir, o) = g16(buf, o);
+ (e.diskcdirentries, o) = g16(buf, o);
+ (e.cdirentries, o) = g16(buf, o);
+ (e.cdirsize, o) = g32(buf, o);
+ (e.cdiroffset, o) = g32(buf, o);
+ (e.comment, o) = gstr(buf, o);
+ if(o != len buf)
+ say(sprint("%d trailing bytes after end of central directory", len buf-o));+ } exception {+ "get:*" =>
+ return (nil, sprint("short buffer for end of central directory buffer, (o %d, len %d)", o, len buf));+ }
+ return (e, nil);
+}
+
+Endofcdir.pack(e: self ref Endofcdir): array of byte
+{+ buf := array[4+4*2+2*4+2+len e.comment] of byte;
+ o := 0;
+ o = pbuf(buf, o, eocentraldirsig);
+ o = p16(buf, o, e.disknr);
+ o = p16(buf, o, e.diskcdir);
+ o = p16(buf, o, e.diskcdirentries);
+ o = p16(buf, o, e.cdirentries);
+ o = p32(buf, o, e.cdirsize);
+ o = p32(buf, o, e.cdiroffset);
+ o = p16(buf, o, len e.comment);
+ o = pbuf(buf, o, e.comment);
+ return buf;
+}
+
+Endofcdir.text(e: self ref Endofcdir): string
+{+ return sprint("Endofcdir(disk: nr %d, cdir %d, cdirentries %d; cdir: entries %d, size %bd, offset %bd; comment %q)",+ e.disknr, e.diskcdir, e.diskcdirentries, e.cdirentries, e.cdirsize, e.cdiroffset, string e.comment);
+}
+
+
+comprmethod(m: int): string
+{+ if(m < 0 || m >= len comprmethods || comprmethods[m] == nil)
+ return "unknown";
+ return comprmethods[m];
+}
+
+open(fd: ref Sys->FD): (ref Endofcdir, array of ref CDFhdr, string)
+{+ {+ return open0(fd);
+ } exception e {+ "open0:*" =>
+ return (nil, nil, e[len "open0:":]);
+ }
+}
+
+error(s: string)
+{+ raise "open0:"+s;
+}
+
+open0(fd: ref Sys->FD): (ref Endofcdir, array of ref CDFhdr, string)
+{+ (ok, dir) := sys->fstat(fd);
+ if(ok < 0)
+ error(sprint("stat: %r"));+ size := dir.length;
+ off := size-big (8*1024);
+ if(off < big 0)
+ off = big 0;
+ n := preadn(fd, buf := array[int (size-off)] of byte, len buf, off);
+ if(n < 0)
+ error(sprint("read: %r"));+ buf = buf[:n];
+ (o, eerr) := findeocdir(buf);
+ if(eerr != nil)
+ error("cannot parse file: "+eerr);+ off += big o;
+
+ (eocdir, err) := Endofcdir.parse(buf[o:]);
+ if(err != nil)
+ error("parsing end of central directory: "+err);+
+ if(eocdir.disknr != 0 || eocdir.diskcdirentries != eocdir.cdirentries)
+ error("split zip file, not supported");+
+ b := bufio->fopen(fd, Bufio->OREAD);
+ if(b == nil)
+ error(sprint("fopen: %r"));+ if(b.seek(eocdir.cdiroffset, Bufio->SEEKSTART) != eocdir.cdiroffset)
+ error(sprint("seek to central directory: %r"));+ a := array[eocdir.cdirentries] of ref CDFhdr;
+ for(i := 0; i < len a; i++) {+ (fhdr, ferr) := CDFhdr.read(b);
+ if(ferr != nil)
+ error("reading central directory file header: "+ferr);+ a[i] = fhdr;
+ }
+
+ return (eocdir, a, nil);
+}
+
+findeocdir(buf: array of byte): (int, string)
+{+ for(o := len buf-(4+2+2+2+2+4+4+2); o >= 0; o--)
+ if(buf[o] == byte 'P' && bufcmp(buf[o:o+4], eocentraldirsig) == 0)
+ return (o, nil);
+ return (-1, "cannot find end of central directory");
+}
+
+supported(f: ref Fhdr): string
+{+ if((f.versneeded & 255) > Version)
+ return sprint("version too low for opening file, have %s, need %s", versstr(Version), versstr(f.versneeded & 255));+ if(f.flags & Fcompressedpatched)
+ return "file is a patch, not supported";
+ if(f.flags & Fstrongcrypto)
+ return "file is new-style encrypted, not supported";
+ if(f.flags & Fencrypted)
+ return "file is encrypted, not supported";
+ return nil;
+}
+
+openfile(fd: ref Sys->FD, cdf: ref CDFhdr): (ref Sys->FD, ref Fhdr, string)
+{+ (f, err) := Fhdr.read(fd, cdf.reloffset);
+ if(err != nil)
+ return (nil, nil, err);
+
+ err = supported(f);
+ if(err != nil)
+ return (nil, nil, err);
+
+ zfd: ref Sys->FD;
+ case f.comprmethod {+ Mplain =>
+ zfd = pushbuf(fd, f.dataoff, int f.uncomprsize, cdf.crc32);
+ Mdeflate =>
+ # xxx +1 is a hack to prevent "premature end of stream" from our inflate. is our inflate broken or is it info-zip 3.0?
+ zfd = pushfilter(fd, f.dataoff, 1+int f.comprsize, cdf.crc32);
+ * =>
+ return (nil, nil, sprint("compression method %q not supported", comprmethod(f.comprmethod)));+ }
+ if(zfd == nil)
+ return (nil, nil, "opening file in zip failed");
+ return (zfd, f, nil);
+}
+
+readfhdr(fd: ref Sys->FD, cdf: ref CDFhdr): (ref Fhdr, string)
+{+ (f, err) := Fhdr.read(fd, cdf.reloffset);
+ if(err == nil)
+ err = supported(f);
+ return (f, err);
+}
+
+# no crc32 protection, assumes supportedness has been checked already.
+pread(fd: ref Sys->FD, f: ref Fhdr, buf: array of byte, n: int, off: big): int
+{+ if(f.comprmethod != Mplain) {+ sys->werrstr("file is not plain (uncompressed)");+ return -1;
+ }
+ if(off > f.uncomprsize)
+ off = f.uncomprsize;
+ if(off+big n > f.uncomprsize)
+ n = int (f.uncomprsize-off);
+ return sys->pread(fd, buf, n, f.dataoff+off);
+}
+
+cvtstr(d: array of byte, isutf8: int): string
+{+ if(isutf8)
+ return string d;
+ return cp437->btos(Convcs->Startstate, d, -1).t1;
+}
+
+versstr(v: int): string
+{+ v &= 255;
+ return sprint("%d.%d", v/10, v%10);+}
+
+filegen: int;
+fileio(): (ref Sys->FD, ref Sys->FileIO)
+{+ f := sprint("f%d", filegen++);+ fio := sys->file2chan("/chan", f);+ if(fio != nil)
+ rfd := sys->open("/chan/"+f, Sys->OREAD);+ return (rfd, fio);
+}
+
+# as long as reads are sequential, keep track of the crc and verify at eof
+pushbuf0(fd: ref Sys->FD, off: big, n: int, fdc: chan of ref Sys->FD, hdrcrc: big)
+{+ (rfd, fio) := fileio();
+ fdc <-= rfd;
+ if(rfd == nil)
+ return;
+
+ prevoff := 0;
+ docrc := 1;
+ crc := ~0;
+ end := off+big n;
+Fio:
+ for(;;) {+ (roff0, count, nil, rc) := <-fio.read;
+ if(rc == nil)
+ return;
+ roff := off+big roff0;
+ rend := roff+big count;
+ if(roff < off)
+ roff = off;
+ if(rend > end)
+ rend = end;
+ nn := sys->pread(fd, buf := array[int (rend-roff)] of byte, len buf, roff);
+ if(nn < 0) {+ rc <-= (nil, sprint("%r"));+ continue;
+ }
+ docrc = docrc && prevoff == roff0;
+ if(docrc) {+ crc = crc32(crc, buf[:nn]);
+ if(nn == 0 && ~crc != int hdrcrc) {+ rc <-= (nil, sprint("crc mismatch, expected %bux, calculated %ux", hdrcrc, ~crc));+ break Fio;
+ }
+ prevoff += nn;
+ }
+ rc <-= (buf[:nn], nil);
+ }
+}
+
+pushbuf(fd: ref Sys->FD, off: big, n: int, hdrcrc: big): ref Sys->FD
+{+ spawn pushbuf0(fd, off, n, fdc := chan of ref Sys->FD, hdrcrc);
+ return <-fdc;
+}
+
+pushfilter0(fd: ref Sys->FD, off: big, n: int, fdc: chan of ref Sys->FD, hdrcrc: big)
+{+ (rfd, fio) := fileio();
+ fdc <-= rfd;
+ if(rfd == nil)
+ return;
+
+ rqc := inflate->start("");+ pid: int;
+ pick srq := <-rqc {+ Start =>
+ pid = srq.pid;
+ * =>
+ return;
+ }
+
+ crc := ~0;
+ poff := 0; # previous offset read
+ buf := array[0] of byte;
+Fio:
+ for(;;) {+ (roff, count, nil, rc) := <-fio.read;
+ if(rc == nil)
+ break Fio;
+ if(roff != poff) {+ rc <-= (nil, "random reads not allowed");
+ break Fio;
+ }
+
+ Filter:
+ while(len buf == 0)
+ pick rq := <-rqc {+ Start =>
+ rc <-= (nil, "bogus start message from filter");
+ break Fio;
+ Fill =>
+ give := len rq.buf;
+ if(give > n)
+ give = n;
+ nn := sys->pread(fd, rq.buf, give, off);
+ rq.reply <-= nn;
+ if(nn < 0) {+ rc <-= (nil, sprint("read: %r"));+ break Fio;
+ }
+ off += big nn;
+ n -= nn;
+ Result =>
+ crc = crc32(crc, rq.buf);
+ buf = array[len rq.buf] of byte;
+ buf[:] = rq.buf;
+ rq.reply <-= 0;
+ break Filter;
+ Finished =>
+ if(len rq.buf != 0)
+ say(sprint("%d leftover bytes", len rq.buf));+ crc = ~crc;
+ if(crc != int hdrcrc) {+ rc <-= (nil, sprint("crc mismatch, expected %bux, calculated %ux", hdrcrc, crc));+ break Fio;
+ }
+ rc <-= (array[0] of byte, nil);
+ break Fio;
+ Info =>
+ say("inflate: "+rq.msg);+ Error =>
+ rc <-= (nil, rq.e);
+ break Fio;
+ }
+
+ give := count;
+ if(give > len buf)
+ give = len buf;
+ r := buf[:give];
+ buf = buf[give:];
+ poff += give;
+ rc <-= (r, nil);
+ }
+ kill(pid);
+}
+
+pushfilter(fd: ref Sys->FD, off: big, n: int, hdrcrc: big): ref Sys->FD
+{+ spawn pushfilter0(fd, off, n, fdc := chan of ref Sys->FD, hdrcrc);
+ return <-fdc;
+}
+
+
+bufcmp(a, b: array of byte): int
+{+ for(i := 0; i < len a; i++)
+ if(a[i] != b[i])
+ return int a[i]-int b[i];
+ return 0;
+}
+
+g16(d: array of byte, o: int): (int, int)
+{+ if(o+2 > len d)
+ raise "get:short buffer";
+ v := 0;
+ v |= int d[o++]<<0;
+ v |= int d[o++]<<8;
+ return (v, o);
+}
+
+g32(d: array of byte, o: int): (big, int)
+{+ if(o+2 > len d)
+ raise "get:short buffer";
+ v := big 0;
+ v |= big d[o++]<<0;
+ v |= big d[o++]<<8;
+ v |= big d[o++]<<16;
+ v |= big d[o++]<<24;
+ return (v, o);
+}
+
+gstr(d: array of byte, o: int): (array of byte, int)
+{+ n: int;
+ (n, o) = g16(d, o);
+ if(o+n > len d)
+ raise "get:short buffer for string";
+ buf := array[n] of byte;
+ buf[:] = d[o:o+n];
+ return (buf, o+n);
+}
+
+gbuf(n: int, d: array of byte, o: int): (array of byte, int)
+{+ if(o+n > len d)
+ raise "get:short buffer for buffer";
+ buf := array[n] of byte;
+ buf[:] = d[o:o+n];
+ return (buf, o+n);
+}
+
+gbufstr(isutf8: int, n: int, d: array of byte, o: int): (string, int)
+{+ buf: array of byte;
+ (buf, o) = gbuf(n, d, o);
+ return (cvtstr(buf, isutf8), o);
+}
+
+p16(d: array of byte, o: int, v: int): int
+{+ d[o++] = byte (v>>0);
+ d[o++] = byte (v>>8);
+ return o;
+}
+
+p32(d: array of byte, o: int, v: big): int
+{+ d[o++] = byte (v>>0);
+ d[o++] = byte (v>>8);
+ d[o++] = byte (v>>16);
+ d[o++] = byte (v>>24);
+ return o;
+}
+
+pbuf(d: array of byte, o: int, buf: array of byte): int
+{+ d[o:] = buf;
+ return o+len buf;
+}
+
+preadn(fd: ref Sys->FD, buf: array of byte, n: int, off: big): int
+{+ have := 0;
+ for(;;) {+ nn := sys->pread(fd, buf[have:], n, off);
+ if(nn < 0)
+ return nn;
+ if(nn == 0)
+ break;
+ have += nn;
+ off += big nn;
+ n -= nn;
+ }
+ return have;
+}
+
+breadn(b: ref Iobuf, buf: array of byte, n: int): int
+{+ have := 0;
+ for(;;) {+ nn := b.read(buf[have:], n-have);
+ if(nn < 0)
+ return nn;
+ if(nn == 0)
+ break;
+ have += nn;
+ }
+ return have;
+}
+
+
+mkcrcval(poly, c: int): int
+{+ for(j := 0; j < 8; j++)
+ if(c & 1)
+ c = poly ^ ((c>>1) & 16r7fffffff);
+ else
+ c = (c>>1) & 16r7fffffff;
+ return c;
+}
+
+mkcrctab(poly: int): array of int
+{+ tab := array[256] of int;
+ for(i := 0; i < 256; i++)
+ tab[i] = mkcrcval(poly, i);
+ return tab;
+}
+
+crc32(crc: int, buf: array of byte): int
+{+ n := len buf;
+ for(i := 0; i < n; i++)
+ crc = crc32tab[(crc ^ int buf[i]) & 255] ^ ((crc>>8) & 16rffffff);
+ return crc;
+}
+
+
+rev[T](l: list of T): list of T
+{+ r: list of T;
+ for(; l != nil; l = tl l)
+ r = hd l::r;
+ return r;
+}
+
+kill(pid: int)
+{+ sys->fprint(sys->open(sprint("/prog/%d/ctl", pid), Sys->OWRITE), "kill");+}
+
+
+say(s: string)
+{+ if(dflag)
+ sys->fprint(sys->fildes(2), "zip: %s\n", s);
+}
--- a/lib/emptydirs
+++ b/lib/emptydirs
@@ -186,3 +186,4 @@
dis/usb
dis/wm
dis/wm/brutus
+dis/zip
--- /dev/null
+++ b/man/1/getzip
@@ -1,0 +1,104 @@
+.TH GETZIP 1
+.SH NAME
+getzip, lszip, putzip \- zip file utilities
+.SH SYNOPSIS
+.B zip/getzip
+[
+.B -dv
+] [
+.B -k
+]
+.I zipfile
+[
+.I path
+.I ...
+]
+.br
+.B zip/lszip
+[
+.B -d
+]
+.I zipfile
+.br
+.B zip/putzip
+[
+.I -dvp
+]
+.I zipfile
+[
+.I path
+.I ...
+]
+.SH DESCRIPTION
+.BR Getzip ,
+.B lszip
+and
+.B putzip
+read, list and create zip files.
+.I Zipfile
+is the file to be read or created.
+.PP
+.B Getzip
+extracts only the
+.I paths
+given on the command-line. If no paths are specified, all files in the zip file are extracted.
+.PP
+.B Lszip
+lists the files present in the zip file.
+.PP
+.B Putzip
+creates a zip file.
+If
+.I paths
+are given on the command-line, those paths and their children are added to the zip file.
+Otherwise, a list of paths to put in the zip file are read from
+standard input. Directories from the standard input are not added
+recursively.
+.TP
+
+.PP
+Options
+.PP
+.TP
+.B -d
+Print debugging information. A second
+.B -d prints more information.
+.TP
+.B -v
+Be verbose.
+For
+.B getzip
+this prints the files extracted.
+For
+.B putzip
+this prints the files added to the zip file.
+.TP
+.B -k
+Keep existing files. For
+.B getzip
+only.
+By default, existing files are overwritten.
+.TP
+.B -p
+Do not compress files added to the zip file. For
+.B putzip
+only.
+
+.SH SOURCE
+.B /appl/cmd/zip/getzip.b
+.br
+.B /appl/cmd/zip/lszip.b
+.br
+.B /appl/cmd/zip/putzip.b
+.br
+.B /appl/lib/zip.b
+.br
+.B /appl/lib/zip.m
+.SH SEE ALSO
+.IR gettar (1),
+.IR zipstream (1),
+.IR zipfs (4)
+.SH BUGS
+Zip64 extensions are not supported.
+.br
+Encrypted zip files are not supported.
--- /dev/null
+++ b/man/1/zipstream
@@ -1,0 +1,39 @@
+.TH ZIPSTREAM 1
+.SH NAME
+zipstream \- rewrite zip file for streaming
+.SH SYNOPSIS
+.B zip/zipstream
+[
+.B -d
+] [
+.B -o
+]
+.I zipfile
+.SH DESCRIPTION
+.B Zipstream
+writes
+.I zipfile
+to standard output, placing the central directory (listing all files and their details) at the start of the file instead of at the end.
+.PP
+If
+.B -o
+is specified, a list of files is read from standard input (separated by newline) and those files moved to the start of the new zip file.
+Files must be specified at most once.
+Files not specified will remain in the same order, but after the files that were specified.
+.PP
+Option
+.B -d
+increases debugging output.
+.SH SOURCE
+.B /appl/cmd/zip/zipstream.b
+.br
+.B /appl/lib/zip.b
+.br
+.B /appl/lib/zip.m
+.SH SEE ALSO
+.IR getzip (1),
+.IR zipfs (4)
+.SH BUGS
+Not all zip utilities can read zip files generated by zipstream. Most notably Info-ZIP and 7z.
+.PP
+The ``end of the central directory'' (pointing to the actual central directory at the beginning) is still at the end of the file.
--- /dev/null
+++ b/man/4/zipfs
@@ -1,0 +1,47 @@
+.TH ZIPFS 4
+.SH NAME
+zipfs \- mount zip archive
+.SH SYNOPSIS
+mount {+.B zip/zipfs
+[
+.B -dDp
+]
+.I zipfile
+}
+mtpt
+.SH DESCRIPTION
+.I Zipfs
+makes the contents of
+.I zipfile
+available in the file system, for reading.
+.I Zipfs
+serves the styx protocol on its file descriptor 0.
+.PP
+Options
+.TP
+.B -d
+Print debugging information. A second
+.B -d
+increases output.
+.TP
+.B -D
+Print a trace of styx messages.
+.TP
+.B -p
+Read uncompressed files in the zip file more efficiently. The downside is that checksums are not checked.
+.SH SOURCE
+.B /appl/cmd/zip/zipfs.b
+.br
+.B /appl/lib/zip.b
+.br
+.B /module/zip.m
+.SH SEE ALSO
+.IR getzip (1),
+.IR zipstream (1).
+.SH BUGS
+Zip64 extensions are not supported.
+.br
+Encrypted archives are not supported.
+.br
+Only uncompressed and deflate-compressed files are supported.
--- /dev/null
+++ b/module/zip.m
@@ -1,0 +1,113 @@
+Zip: module
+{+ PATH: con "/dis/lib/zip.dis";
+
+ dflag: int;
+ init: fn(): string;
+
+ # compression methods. only plain & deflate supported.
+ Mplain,
+ Mshrunk,
+ Mreduced1, Mreduced2, Mreduced3, Mreduced4,
+ Mimplode, Mtokenize,
+ Mdeflate, Mdeflate64,
+ Mpkwareimplode,
+ Mreserved0,
+ Mbzip2,
+ Mreserved1,
+ Mlzma: con iota+0;
+ Mibmterse,
+ Mlz77z: con iota+18;
+ Mwavpack,
+ Mppmdi1: con iota+97;
+
+ # general purpose flags
+ Fencrypted: con 1<<0;
+ Flocaldatadescr: con 1<<3; # crc & sizes in fhdr are 0, use "data descriptor" following fhdr
+ Fcompressedpatched: con 1<<5;
+ Fstrongcrypto: con 1<<6;
+ Futf8: con 1<<11;
+ Fcdirencrypted: con 1<<13;
+
+ # internal file attributes
+ IFArecord: con 1<<1;
+
+ Version: con 20; # supported for reading
+
+ Extra: adt {+ l: list of ref (int, array of byte);
+
+ parse: fn(d: array of byte): (ref Extra, string);
+ pack: fn(e: self ref Extra): array of byte;
+ text: fn(e: self ref Extra): string;
+ };
+
+ Fhdr: adt {+ versneeded: int;
+ flags: int;
+ comprmethod: int;
+ filemtime: int;
+ filemdate: int;
+ mtime: int; # not in file, unix epoch mtime based on filemtime & filemdate
+ crc32: big;
+ comprsize: big;
+ uncomprsize: big;
+ filename: string;
+ extra: ref Extra;
+ dataoff: big; # not in file
+
+ mk: fn(f: ref CDFhdr): ref Fhdr;
+ parse: fn(buf: array of byte, off: big): (ref Fhdr, string);
+ read: fn(fd: ref Sys->FD, off: big): (ref Fhdr, string);
+ pack: fn(f: self ref Fhdr): array of byte;
+ text: fn(f: self ref Fhdr): string;
+ };
+
+ CDFhdr: adt {+ versmadeby: int;
+ versneeded: int;
+ flags: int;
+ comprmethod: int;
+ filemtime: int;
+ filemdate: int;
+ mtime: int; # not in file, unix epoch mtime based on filemtime & filemdate
+ crc32: big;
+ comprsize: big;
+ uncomprsize: big;
+ filename: string;
+ extra: ref Extra;
+ comment: string;
+ disknrstart: int;
+ intattr: int;
+ extattr: big;
+ reloffset: big;
+
+ mk: fn(f: ref Fhdr, off: big): ref CDFhdr;
+ parse: fn(buf: array of byte): (ref CDFhdr, string);
+ read: fn(b: ref Bufio->Iobuf): (ref CDFhdr, string);
+ pack: fn(f: self ref CDFhdr): array of byte;
+ text: fn(f: self ref CDFhdr): string;
+ };
+
+ Endofcdir: adt {+ disknr: int;
+ diskcdir: int;
+ diskcdirentries: int;
+ cdirentries: int;
+ cdirsize: big;
+ cdiroffset: big;
+ comment: array of byte;
+
+ parse: fn(buf: array of byte): (ref Endofcdir, string);
+ pack: fn(e: self ref Endofcdir): array of byte;
+ text: fn(e: self ref Endofcdir): string;
+ };
+
+ comprmethod: fn(m: int): string;
+ open: fn(fd: ref Sys->FD): (ref Endofcdir, array of ref CDFhdr, string);
+ openfile: fn(fd: ref Sys->FD, f: ref CDFhdr): (ref Sys->FD, ref Fhdr, string);
+ readfhdr: fn(fd: ref Sys->FD, f: ref CDFhdr): (ref Fhdr, string);
+ pread: fn(fd: ref Sys->FD, f: ref Fhdr, buf: array of byte, n: int, off: big): int;
+ sanitizepath: fn(s: string): string;
+ crc32: fn(crc: int, buf: array of byte): int;
+};