diff options
Diffstat (limited to 'appl/svc/webget')
| -rw-r--r-- | appl/svc/webget/date.b | 266 | ||||
| -rw-r--r-- | appl/svc/webget/date.m | 12 | ||||
| -rw-r--r-- | appl/svc/webget/file.b | 67 | ||||
| -rw-r--r-- | appl/svc/webget/ftp.b | 227 | ||||
| -rw-r--r-- | appl/svc/webget/http.b | 602 | ||||
| -rw-r--r-- | appl/svc/webget/image2enc.b | 1070 | ||||
| -rw-r--r-- | appl/svc/webget/image2enc.m | 7 | ||||
| -rw-r--r-- | appl/svc/webget/message.b | 249 | ||||
| -rw-r--r-- | appl/svc/webget/message.m | 28 | ||||
| -rw-r--r-- | appl/svc/webget/mkfile | 40 | ||||
| -rw-r--r-- | appl/svc/webget/transport.m | 5 | ||||
| -rw-r--r-- | appl/svc/webget/webget.b | 464 | ||||
| -rw-r--r-- | appl/svc/webget/webget.log | 0 | ||||
| -rw-r--r-- | appl/svc/webget/wgutils.b | 298 | ||||
| -rw-r--r-- | appl/svc/webget/wgutils.m | 53 |
15 files changed, 3388 insertions, 0 deletions
diff --git a/appl/svc/webget/date.b b/appl/svc/webget/date.b new file mode 100644 index 00000000..71248954 --- /dev/null +++ b/appl/svc/webget/date.b @@ -0,0 +1,266 @@ +implement Date; + +include "sys.m"; + sys: Sys; + +include "daytime.m"; + daytime : Daytime; + +Tm: import daytime; + +include "date.m"; + + # print dates in the format + # Wkd, DD Mon YYYY HH:MM:SS GMT + # parse dates of formats + # Wkd, DD Mon YYYY HH:MM:SS GMT + # Weekday, DD-Mon-YY HH:MM:SS GMT + # Wkd Mon ( D|DD) HH:MM:SS YYYY + # plus anything similar + +SEC2MIN: con 60; +SEC2HOUR: con (60*SEC2MIN); +SEC2DAY: con (24*SEC2HOUR); + +# days per month plus days/year + +dmsize := array[] of { + 365, 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31 +}; + +ldmsize := array[] of { + 366, 31, 29, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31 +}; + + +# return the days/month for the given year + + +weekdayname := array[] of { + "Sunday", "Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday" +}; + +wdayname := array[] of { + "Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat" +}; + + +monname := array[] of { + "Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec" +}; + +init() +{ + daytime = load Daytime Daytime->PATH; + sys = load Sys "$Sys"; + if (daytime==nil) + sys->print("daytime load: %r\n"); +} + +# internals.... +dateindex : fn(nil: string, nill:array of string): int; +gmtm2sec : fn(tm: Tm): int; + + +yrsize(yr : int): array of int { + if(yr % 4 == 0 && (yr % 100 != 0 || yr % 400 == 0)) + return ldmsize; + else + return dmsize; +} + +tolower(c: int): int { + if(c >= 'A' && c <= 'Z') + return c - 'A' + 'a'; + return c; +} + + +isalpha(c: int): int{ + return c >= 'A' && c <= 'Z' || c >= 'a' && c <= 'z'; +} + + +isdig(c: int): int { + return c >= '0' && c <= '9'; +} + + +dateconv(t: int): string { + tm : ref Tm; + tm = daytime->gmt(t); + return sys->sprint("%s, %.2d %s %.4d %.2d:%.2d:%.2d GMT", + wdayname[tm.wday], tm.mday, monname[tm.mon], tm.year+1900, + tm.hour, tm.min, tm.sec); +} + + +dateword(date : string): (string,string) { + p : string; + i:=0; + p = ""; + while((i<len date) && !isalpha(date[i]) && !isdig(date[i])) + i++; + while((i<len date) && isalpha(date[i])){ + p[len p] = tolower(date[i]); + i++; + } + rest := ""; + if(i < len date) + rest = date[i:]; + return (rest,p); +} + + +datenum(date : string): (string, int){ + n, i : int; + i=0; + while((i<len date) && !isdig(date[i])) + i++; + if(i == len date) + return (nil, -1); + n = 0; + while((i<len date) && isdig(date[i])){ + n = n * 10 + date[i] - '0'; + i++; + } + return (date[i:], n); +} + + + # parse a date and return the seconds since the epoch + # return 0 for a failure + +# could be big? +date2sec(date : string): int { + tm : Tm; + buf : string; + + # Weekday|Wday + + (date,buf) = dateword(date); + tm.wday = dateindex(buf, wdayname); + if(tm.wday < 0) + tm.wday = dateindex(buf, weekdayname); + + if(tm.wday < 0) + return 0; + + # check for the two major formats + + (date,buf) = dateword(date); + tm.mon = dateindex(buf, monname); + if(tm.mon >= 0){ + # MM + (date, tm.mday) = datenum(date); + if(tm.mday < 1 || tm.mday > 31) + return 0; + + # HH:MM:SS + (date, tm.hour) = datenum(date); + if(tm.hour < 0 || tm.hour >= 24) + return 0; + (date, tm.min) = datenum(date); + if(tm.min < 0 || tm.min >= 60) + return 0; + (date, tm.sec) = datenum(date); + if(tm.sec < 0 || tm.sec >= 60) + return 0; + + + # YYYY + (nil, tm.year) = datenum(date); + if(tm.year < 70 || tm.year > 99 && tm.year < 1970) + return 0; + if(tm.year >= 1970) + tm.year -= 1900; + }else{ + # MM-Mon-(YY|YYYY) + (date, tm.mday) = datenum(date); + if(tm.mday < 1 || tm.mday > 31) + return 0; + (date,buf) = dateword(date); + tm.mon = dateindex(buf, monname); + if(tm.mon < 0 || tm.mon >= 12) + return 0; + (date, tm.year) = datenum(date); + if(tm.year < 70 || tm.year > 99 && tm.year < 1970) + return 0; + if(tm.year >= 1970) + tm.year -= 1900; + + # HH:MM:SS + (date, tm.hour) = datenum(date); + if(tm.hour < 0 || tm.hour >= 24) + return 0; + (date, tm.min) = datenum(date); + if(tm.min < 0 || tm.min >= 60) + return 0; + (date, tm.sec) = datenum(date); + if(tm.sec < 0 || tm.sec >= 60) + return 0; + + # timezone + (date,buf)=dateword(date); + if(len buf >= 3 && lowercase(buf[0:3])!="gmt") + return 0; + } + + tm.zone="GMT"; + return gmtm2sec(tm); +} + +lowercase(name:string): string { + p: string; + for(i:=0;i<len name;i++) + p[i]=tolower(name[i]); + return p; +} + +dateindex(d : string, tab : array of string): int{ + for(i := 0; i < len tab; i++) + if (lowercase(tab[i]) == d) + return i; + return -1; +} + + +# compute seconds since Jan 1 1970 GMT + +gmtm2sec(tm:Tm): int +{ + secs,i : int; + d2m: array of int; + sys = load Sys "$Sys"; + secs=0; + + #seconds per year + tm.year += 1900; + if(tm.year < 1970) + return 0; + for(i = 1970; i < tm.year; i++){ + d2m = yrsize(i); + secs += d2m[0] * SEC2DAY; + } + + + #seconds per month + d2m = yrsize(tm.year); + for(i = 0; i < tm.mon; i++) + secs += d2m[i+1] * SEC2DAY; + + #secs in last month + secs += (tm.mday-1) * SEC2DAY; + + #hours, minutes, seconds + secs += tm.hour * SEC2HOUR; + secs += tm.min * SEC2MIN; + secs += tm.sec; + + return secs; +} + +now(): int +{ + return daytime->now(); +} diff --git a/appl/svc/webget/date.m b/appl/svc/webget/date.m new file mode 100644 index 00000000..61c31a48 --- /dev/null +++ b/appl/svc/webget/date.m @@ -0,0 +1,12 @@ + +Date: module{ + PATH : con "/dis/svc/webget/date.dis"; + + dateconv: fn(secs :int): string; # returns an http formatted + # date representing secs. + date2sec: fn(foo:string): int; # parses a date and returns + # number of secs since the + # epoch that it represents. + now: fn(): int; # so don't have to load daytime too + init: fn(); # to load needed modules +}; diff --git a/appl/svc/webget/file.b b/appl/svc/webget/file.b new file mode 100644 index 00000000..49c51423 --- /dev/null +++ b/appl/svc/webget/file.b @@ -0,0 +1,67 @@ +implement Transport; + +include "sys.m"; + sys: Sys; + +include "draw.m"; + +include "string.m"; + S: String; + +include "bufio.m"; + B : Bufio; + Iobuf: import Bufio; + +include "message.m"; + M: Message; + Msg, Nameval: import M; + +include "url.m"; + U: Url; + ParsedUrl: import U; + +include "webget.m"; + +include "wgutils.m"; + W: WebgetUtils; + Fid, Req: import WebgetUtils; + +include "transport.m"; + +init(w: WebgetUtils) +{ + sys = load Sys Sys->PATH; + W = w; + M = W->M; + S = W->S; + B = W->B; + U = W->U; +} + +connect(c: ref Fid, r: ref Req, donec: chan of ref Fid) +{ + u := r.url; + mrep: ref Msg = nil; + if(!(u.host == "" || u.host == "localhost")) + mrep = W->usererr(r, "no remote file system to " + u.host); + else { + f := u.pstart + u.path; + io := B->open(f, sys->OREAD); + if(io == nil) + mrep = W->usererr(r, sys->sprint("can't open %s: %r\n", f)); + else { + mrep = Msg.newmsg(); + e := W->getdata(io, mrep, W->fixaccept(r.types), u); + B->io.close(); + if(e != "") + mrep = W->usererr(r, e); + else + W->okprefix(r, mrep); + } + } + if(mrep != nil) { + W->log(c, "file: reply ready for " + r.reqid + ": " + mrep.prefixline); + r.reply = mrep; + donec <-= c; + } +} diff --git a/appl/svc/webget/ftp.b b/appl/svc/webget/ftp.b new file mode 100644 index 00000000..9162fcb7 --- /dev/null +++ b/appl/svc/webget/ftp.b @@ -0,0 +1,227 @@ +implement Transport; + +include "sys.m"; + sys: Sys; + +include "draw.m"; + +include "string.m"; + S: String; + +include "bufio.m"; + B : Bufio; + Iobuf: import Bufio; + +include "message.m"; + M: Message; + Msg, Nameval: import M; + +include "url.m"; + U: Url; + ParsedUrl: import U; + +include "webget.m"; + +include "wgutils.m"; + W: WebgetUtils; + Fid, Req: import WebgetUtils; + +include "transport.m"; + +FTPPORT: con "21"; +DEBUG: con 1; + +# Return codes +Extra, Success, Incomplete, TempFail, PermFail : con (1+iota); + +init(w: WebgetUtils) +{ + sys = load Sys Sys->PATH; + W = w; + M = W->M; + S = W->S; + B = W->B; + U = W->U; +} + +connect(c: ref Fid, r: ref Req, donec: chan of ref Fid) +{ + mrep: ref Msg = nil; + io, dio: ref Iobuf = nil; + err := ""; + u := r.url; + port := u.port; + if(port == "") + port = FTPPORT; + addr := "tcp!" + u.host + "!" + port; + +dummyloop: # just for breaking out of on error + for(;;) { + W->log(c, sys->sprint("ftp: dialing %s", addr)); + (ok, net) := sys->dial(addr, nil); + if(ok < 0) { + err = sys->sprint("dial error: %r"); + break dummyloop; + } + io = B->fopen(net.dfd, sys->ORDWR); + if(io == nil) { + err = "cannot open network via bufio"; + break dummyloop; + } + + # look for Hello + (code, msg) := getreply(c, io); + if(code != Success) { + err = "instead of hello: " + msg; + break dummyloop; + } + # logon + err = sendrequest(c, io, "USER anonymous"); + if(err != "") + break dummyloop; + (code, msg) = getreply(c, io); + if(code != Success) { + if(code == Incomplete) { + # need password + err = sendrequest(c, io, "PASS webget@webget.com"); + (code, msg) = getreply(c, io); + if(code != Success) { + err = "login failed: " + msg; + break dummyloop; + } + } + else { + err = "login failed: " + msg; + break dummyloop; + } + } + # image type + err = sendrequest(c, io, "TYPE I"); + (code, msg) = getreply(c, io); + if(code != Success) { + err = "can't set type I: " + msg; + break dummyloop; + } + # passive mode + err = sendrequest(c, io, "PASV"); + (code, msg) = getreply(c, io); + if(code != Success) { + err = "can't use passive mode: " + msg; + break dummyloop; + } + (paddr, pport) := passvap(msg); + if(paddr == "") { + err = "passive mode protocol botch: " + msg; + break dummyloop; + } + # dial data port + daddr := "tcp!" + paddr + "!" + pport; + W->log(c, sys->sprint("ftp: dialing data %s", daddr)); + (ok2, dnet) := sys->dial(daddr, nil); + if(ok2 < 0) { + err = sys->sprint("data dial error: %r"); + break dummyloop; + } + dio = B->fopen(dnet.dfd, sys->ORDWR); + if(dio == nil) { + err = "cannot open network via bufio"; + break dummyloop; + } + # tell remote to send file + err = sendrequest(c, io, "RETR " + u.path); + (code, msg) = getreply(c, io); + if(code != Extra) { + err = "passive mode retrieve failed: " + msg; + break dummyloop; + } + + mrep = Msg.newmsg(); +W->log(c, "reading from dio now"); + err = W->getdata(dio, mrep, W->fixaccept(r.types), u); +W->log(c, "done reading from dio now, err=" + err); + B->dio.close(); + if(err == "") + W->okprefix(r, mrep); + break dummyloop; + } + if(io != nil) + B->io.close(); + if(dio != nil) + B->dio.close(); + if(err != "") + mrep = W->usererr(r, err); + if(mrep != nil) { + W->log(c, "ftp: reply ready for " + r.reqid + ": " + mrep.prefixline); + r.reply = mrep; + donec <-= c; + } +} + +getreply(c: ref Fid, io: ref Iobuf) : (int, string) +{ + for(;;) { + line := B->io.gets('\n'); + n := len line; + if(n == 0) + break; + if(DEBUG) + W->log(c, "ftp: got reply: " + line); + if(line[n-1] == '\n') { + if(n > 2 && line[n-2] == '\r') + line = line[0:n-2]; + else + line = line[0:n-1]; + } + rv := int line; + if(rv >= 100 && rv < 600) { + # if line is like '123-stuff' + # then there will be more lines until + # '123 stuff' + if(len line<4 || line[3]==' ') + return (rv/100, line); + } + } + return (-1, ""); +} + +sendrequest(c: ref Fid, io: ref Iobuf, cmd: string) : string +{ + if(DEBUG) + W->log(c, "ftp: send request: " + cmd); + cmd = cmd + "\r\n"; + buf := array of byte cmd; + n := len buf; + if(B->io.write(buf, n) != n) + return sys->sprint("write error: %r"); + return ""; +} + +passvap(s: string) : (string, string) +{ + # Parse reply to PASSV to find address and port numbers. + # This is AI + addr := ""; + port := ""; + (nil, v) := S->splitl(s, "("); + if(v != "") + s = v[1:]; + else + (nil, s) = S->splitl(s, "0123456789"); + if(s != "") { + (n, l) := sys->tokenize(s, ","); + if(n >= 6) { + addr = hd l + "."; + l = tl l; + addr += hd l + "."; + l = tl l; + addr += hd l + "."; + l = tl l; + addr += hd l; + l = tl l; + p1 := int hd l; + p2 := int hd tl l; + port = string (((p1&255)<<8)|(p2&255)); + } + } + return (addr, port); +} diff --git a/appl/svc/webget/http.b b/appl/svc/webget/http.b new file mode 100644 index 00000000..35b52966 --- /dev/null +++ b/appl/svc/webget/http.b @@ -0,0 +1,602 @@ +implement Transport; + +include "sys.m"; + sys: Sys; + +include "draw.m"; + +include "string.m"; + S: String; + +include "bufio.m"; + B : Bufio; + Iobuf: import B; + +include "date.m"; + D: Date; + +include "message.m"; + M: Message; + Msg, Nameval: import M; + +include "url.m"; + U: Url; + ParsedUrl: import U; + +include "webget.m"; + +include "wgutils.m"; + W: WebgetUtils; + Fid, Req: import WebgetUtils; + +include "keyring.m"; +include "asn1.m"; +include "pkcs.m"; +include "sslsession.m"; +include "ssl3.m"; + ssl3: SSL3; + Context: import ssl3; +# Inferno supported cipher suites: RSA_EXPORT_RC4_40_MD5 +ssl_suites := array [] of {byte 0, byte 16r03}; +ssl_comprs := array [] of {byte 0}; + +include "transport.m"; + +HTTPD: con "80"; # Default IP port +HTTPSD: con "443"; # Default IP port for HTTPS +Version: con "1.0"; # Client ID +MAXREDIR: con 10; + +HTTPheader: adt +{ + vers: string; + code: int; + length: int; + content: string; +}; + +Resp: adt +{ + code: int; + action: int; + cacheable: int; + name: string; +}; + +DODATA, ERROR, REDIR, UNAUTH, HTMLERR: con iota; + +usecache := 1; +cachedir: con "/services/webget/cache"; + +httpproxy: ref ParsedUrl; +agent := "Inferno-webget/" + Version; + +responses := array[] of { + (Resp)(100, DODATA, 0, "Continue" ), + (Resp)(101, ERROR, 0, "Switching Protocols" ), + (Resp)(200, DODATA, 1, "Ok" ), + (Resp)(201, DODATA, 0, "Created" ), + (Resp)(202, DODATA, 0, "Accepted" ), + (Resp)(203, DODATA, 1, "Non-Authoratative Information" ), + (Resp)(204, DODATA, 0, "No content" ), + (Resp)(205, DODATA, 0, "Reset content" ), + (Resp)(206, DODATA, 0, "Partial content" ), + (Resp)(300, ERROR, 1, "Multiple choices" ), + (Resp)(301, REDIR, 1, "Moved permanently" ), + (Resp)(302, REDIR, 0, "Moved temporarily" ), + (Resp)(303, ERROR, 0, "See other" ), + (Resp)(304, ERROR, 0, "Not modified" ), + (Resp)(305, ERROR, 0, "Use proxy" ), + (Resp)(400, HTMLERR, 0, "Bad request" ), + (Resp)(401, UNAUTH, 0, "Unauthorized" ), + (Resp)(402, HTMLERR, 0, "Payment required" ), + (Resp)(403, HTMLERR, 0, "Forbidden" ), + (Resp)(404, HTMLERR, 0, "Not found" ), + (Resp)(405, HTMLERR, 0, "Method not allowed" ), + (Resp)(406, HTMLERR, 0, "Not Acceptable" ), + (Resp)(407, HTMLERR, 0, "Proxy authentication required" ), + (Resp)(408, HTMLERR, 0, "Request timed-out" ), + (Resp)(409, HTMLERR, 0, "Conflict" ), + (Resp)(410, HTMLERR, 1, "Gone" ), + (Resp)(411, HTMLERR, 0, "Length required" ), + (Resp)(412, HTMLERR, 0, "Precondition failed" ), + (Resp)(413, HTMLERR, 0, "Request entity too large" ), + (Resp)(414, HTMLERR, 0, "Request-URI too large" ), + (Resp)(415, HTMLERR, 0, "Unsupported media type" ), + (Resp)(500, ERROR, 0, "Internal server error"), + (Resp)(501, ERROR, 0, "Not implemented"), + (Resp)(502, ERROR, 0, "Bad gateway"), + (Resp)(503, ERROR, 0, "Service unavailable"), + (Resp)(504, ERROR, 0, "Gateway time-out"), + (Resp)(505, ERROR, 0, "HTTP version not supported"), +}; + +init(w: WebgetUtils) +{ + sys = load Sys Sys->PATH; + D = load Date Date->PATH; + D->init(); + W = w; + M = W->M; + S = W->S; + B = W->B; + U = W->U; + ssl3 = nil; # load on demand + readconfig(); +} + +readconfig() +{ + cfgio := B->open("/services/webget/config", sys->OREAD); + if(cfgio != nil) { + for(;;) { + line := B->cfgio.gets('\n'); + if(line == "") { + B->cfgio.close(); + break; + } + if(line[0]=='#') + continue; + (key, val) := S->splitl(line, " \t"); + val = S->take(S->drop(val, " \t"), "^\r\n"); + if(val == "") + continue; + if(key == "httpproxy" && val != "none") { + # val should be host or host:port + httpproxy = U->makeurl("http://" + val); + W->log(nil, "Using http proxy " + httpproxy.tostring()); + usecache = 0; + } + if(key == "agent") { + agent = val; + W->log(nil, sys->sprint("User agent specfied as '%s'\n", agent)); + } + } + } +} + +connect(c: ref Fid, r: ref Req, donec: chan of ref Fid) +{ + method := r.method; + u := r.url; + accept := W->fixaccept(r.types); + mrep, cachemrep: ref Msg = nil; + validate : string; + io: ref Iobuf = nil; + redir := 1; + usessl := 0; + sslx : ref Context; + + redirloop: + for(nredir := 0; redir && nredir < MAXREDIR; nredir++) { + redir = 0; + mrep = nil; + cachemrep = nil; + io = nil; + validate = ""; + if(u.scheme == Url->HTTPS) { + usessl = 1; + if(ssl3 == nil) { + ssl3 = load SSL3 SSL3->PATH; + ssl3->init(); + sslx = ssl3->Context.new(); + } + } + cacheit := usecache; + if(r.cachectl == "no-cache" || usessl) + cacheit = 0; + resptime := 0; + # + # Perhaps try the cache + # + if(usecache && method == "GET") { + (cachemrep, validate) = cacheread(c, u, r); + if(cachemrep != nil && validate == "") + cacheit = 0; + } + else + cacheit = 0; + + if(cachemrep == nil || validate != "") { + # + # Find the port and dial the network + # + dialu := u; + if(httpproxy != nil) + dialu = httpproxy; + port := dialu.port; + if(port == "") { + if(usessl) + port = HTTPSD; + else + port = HTTPD; + } + addr := "tcp!" + dialu.host + "!" + port; + + W->log(c, sys->sprint("http: dialing %s", addr)); + (ok, net) := sys->dial(addr, nil); + if(ok < 0) { + mrep = W->usererr(r, sys->sprint("%r")); + break redirloop; + } + W->log(c, "http: connected"); + e: string; + if(usessl) { + vers := 3; + info := ref SSL3->Authinfo(ssl_suites, ssl_comprs, nil, 0, nil, nil, nil); + (e, vers) = sslx.client(net.dfd, addr, vers, info); + if(e != "") { + mrep = W->usererr(r, e); + break redirloop; + } + W->log(c, "https: ssl handshake complete"); + } + + # + # Issue the request + # + m := Msg.newmsg(); + requ: string; + if(httpproxy != nil) + requ = u.tostring(); + else { + requ = u.pstart + u.path; + if(u.query != "") + requ += "?" + u.query; + } + m.prefixline = method + " " + requ + " HTTP/1.0"; + hdrs := Nameval("Host", u.host) :: + Nameval("User-agent", agent) :: + Nameval("Accept", accept) :: nil; + m.addhdrs(hdrs); + if(validate != "") + m.addhdrs(Nameval("If-Modified_Since", validate) :: nil); + if(r.auth != "") { + m.addhdrs(Nameval("Authorization", "Basic " + r.auth) :: nil); + cacheit = 0; + } + if(method == "POST") { + m.body = r.body; + m.bodylen = len m.body; + m.addhdrs(Nameval("Content-Length", string (len r.body)) :: + Nameval("Content-type", "application/x-www-form-urlencoded") :: + nil); + } + io = B->fopen(net.dfd, sys->ORDWR); + if(io == nil) { + mrep = W->usererr(r, "cannot open network via bufio"); + break redirloop; + } + e = m.writemsg(io); + if(e != "") { + mrep = W->usererr(r, e); + break redirloop; + } + (mrep, e) = Msg.readhdr(io, 1); + if(e!= "") { + mrep = W->usererr(r, e); + break redirloop; + } + resptime = D->now(); + } + else + mrep = cachemrep; + status := mrep.prefixline; + W->log(c, "http: response from network or cache: " + status + + "\n" + mrep.header() + ); + + if(!S->prefix("HTTP/", status)) { + mrep = W->usererr(r, "expected http got something else"); + break redirloop; + } + code := getcode(status); + + if(validate != "" && code == 304) { + # response: "Not Modified", so use cached response + mrep = cachemrep; + B->io.close(); + io = nil; + + # let caching happen with new response time + # (so age will be small next time) + status = mrep.prefixline; + W->log(c, "http: validate ok, using cache: " + status); + code = getcode(status); + } + + for(i := 0; i < len responses; i++) { + if(responses[i].code == code) + break; + } + + if(i >= len responses) { + mrep = W->usererr(r, "Unrecognized HTTP response code"); + break redirloop; + } + + (nil, conttype) := mrep.fieldval("content-type"); + cacheit = cacheit && responses[i].cacheable; + case responses[i].action { + DODATA => + e := W->getdata(io, mrep, accept, u); + if(e != "") + mrep = W->usererr(r, e); + else { + if(cacheit) + cachewrite(c, mrep, u, resptime); + W->okprefix(r, mrep); + } + ERROR => + mrep = W->usererr(r, responses[i].name); + UNAUTH => + (cok, chal) := mrep.fieldval("www-authenticate"); + if(cok && r.auth == "") + mrep = W->usererr(r, "Unauthorized: " + chal); + else { + if(conttype == "text/html" && htmlok(accept)) { + e := W->getdata(io, mrep, accept, u); + if(e != "") + mrep = W->usererr(r, "Authorization needed"); + else + W->okprefix(r, mrep); + } + else + mrep = W->usererr(r, "Authorization needed"); + } + REDIR => + (nil, newloc) := mrep.fieldval("location"); + if(newloc == "") { + e := W->getdata(io, mrep, accept, u); + if(e != "") + mrep = W->usererr(r, e); + else + W->okprefix(r, mrep); + } + else { + if(cacheit) + cachewrite(c, mrep, u, resptime); + if(method == "POST") { + # this is called "erroneous behavior of some + # HTTP 1.0 clients" in the HTTP 1.1 spec, + # but servers out there rely on this... + method = "GET"; + } + oldu := u; + u = U->makeurl(newloc); + u.frag = ""; + u.makeabsolute(oldu); + W->log(c, "http: redirect to " + u.tostring()); + if(io != nil) { + B->io.close(); + io = nil; + } + redir = 1; + } + HTMLERR => + if(cacheit) + cachewrite(c, mrep, u, resptime); + if(conttype == "text/html" && htmlok(accept)) { + e := W->getdata(io, mrep, accept, u); + if(e != "") + mrep = W->usererr(r, responses[i].name); + else + W->okprefix(r, mrep); + } + else + mrep = W->usererr(r, responses[i].name); + } + } + if(io != nil) + B->io.close(); + if(nredir == MAXREDIR) + mrep = W->usererr(r, "redirect loop"); + if(mrep != nil) { + W->log(c, "http: reply ready for " + r.reqid + ": " + mrep.prefixline); + r.reply = mrep; + donec <-= c; + } +} + +getcode(status: string) : int +{ + (vers, scode) := S->splitl(status, " "); + scode = S->drop(scode, " "); + return int scode; +} + +htmlok(accept: string) : int +{ + (nil,y) := S->splitstrl(accept, "text/html"); + return (y != ""); +} + +mkhtml(msg: string) : ref Msg +{ + m := Msg.newmsg(); + m.body = array of byte sys->sprint("<HTML>\n"+ + "<BODY>\n"+ + "<H1>HTTP Reported Error</H1>\n"+ + "<P>\n"+ + "The server reported an error: %s\n"+ + "</BODY>\n"+ + "</HTML>\n", msg); + m.bodylen = len m.body; + m.update("content-type", "text/html"); + m.update("content-location", "webget-internal-message"); + return m; +} + +cacheread(c: ref Fid, u: ref Url->ParsedUrl, r: ref Req) : (ref Msg, string) +{ + ctl := r.cachectl; + if(ctl == "no-cache") + return (nil, ""); + uname := u.tostring(); + hname := hashname(uname); + io := B->open(hname, sys->OREAD); + if(io == nil) + return (nil, ""); + (mrep, e) := Msg.readhdr(io, 1); + if(e != "") { + B->io.close(); + return (nil, ""); + } + + # see if cache validation is necessary + validate := ""; + cmaxstale := 0; + cmaxage := -1; + (nl, l) := sys->tokenize(ctl, ","); + for(i := 0; i < nl; i++) { + s := hd l; + if(S->prefix("max-stale=", s)) + cmaxstale = int s[10:]; + else if (S->prefix("max-age=", s)) + cmaxage = int s[8:]; + l = tl l; + } + # we wrote x-resp-time and x-url, so they should be there + (srst, sresptime) := mrep.fieldval("x-resp-time"); + (su, surl) := mrep.fieldval("x-url"); + if(!srst || !su) { + cacheremove(hname); + B->io.close(); + return (nil, ""); + } + if(surl != uname) { + B->io.close(); + return (nil, ""); + } + (se, sexpires) := mrep.fieldval("expires"); + (sd, sdate) := mrep.fieldval("date"); + (slm, slastmod) := mrep.fieldval("last-modified"); + (sa, sage) := mrep.fieldval("age"); + + # calculate response age (in seconds), as of time received + respt := int sresptime; + datet := D->date2sec(sdate); + nowt := D->now(); + + age := nowt - respt; + if(sa) + age += (int sage); + freshness_lifetime := 0; + (sma, smaxage) := mrep.fieldval("max-age"); + if(sma) + freshness_lifetime = int smaxage; + else if(sd && se) { + exp := D->date2sec(sexpires); + freshness_lifetime = exp - datet; + } + else if(slm){ + # use heuristic: 10% of time since last modified + lastmod := D->date2sec(slastmod); + if(lastmod == 0) + lastmod = respt; + freshness_lifetime = (nowt - lastmod) / 10; + } + if(age - freshness_lifetime > cmaxstale || + (cmaxage != -1 && age >= cmaxage)) { + W->log(c, sys->sprint("must revalidate, age=%d, lifetime=%d, cmaxstale=%d, cmaxage=%d\n", + age, freshness_lifetime, cmaxstale, cmaxage)); + if(slm) + validate = slastmod; + else + return (nil, ""); + } + e = mrep.readbody(io); + B->io.close(); + if(e != "") { + cacheremove(hname); + return (nil, ""); + } + if(validate == "") + W->log(c, "cache hit " + hname); + else + W->log(c, "cache hit " + hname + " if not modified after " + validate); + return (mrep, validate); +} + +cachewrite(c: ref Fid, m: ref Msg, u: ref Url->ParsedUrl, respt: int) +{ + (sp, spragma) := m.fieldval("pragma"); + if(sp && spragma == "no-cache") + return; + (scc, scachectl) := m.fieldval("cache-control"); + if(scc) { + (snc, nil) := attrval(scachectl, "no-cache"); + (sns, nil) := attrval(scachectl, "no-store"); + (smv, nil) := attrval(scachectl, "must-revalidate"); + if(snc || sns || smv) + return; + } + uname := u.tostring(); + hname := hashname(uname); + m.update("x-resp-time", string respt); + m.update("x-url", uname); + m.update("content-length", string m.bodylen); + io := B->create(hname, sys->OWRITE, 8r666); + if(io != nil) { + W->log(c, "cache writeback to " + hname); + m.writemsg(io); + B->io.close(); + } +} + +cacheremove(hname: string) +{ + sys->remove(hname); +} + +attrval(avs, aname: string) : (int, string) +{ + (nl, l) := sys->tokenize(avs, ","); + for(i := 0; i < nl; i++) { + s := hd l; + (lh, rh) := S->splitl(s, "="); + lh = trim(lh); + if(lh == aname) { + if(rh != "") + rh = trim(rh[1:]); + return (1, rh); + } + l = tl l; + } + return (0, ""); +} + +trim(s: string) : string +{ + is := 0; + ie := len s; + while(is < ie) { + if(!S->in(s[is], " \t\n\r")) + break; + is++; + } + if(is == ie) + return ""; + if(s[is] == '"') + is++; + while(ie > is) { + if(!S->in(s[ie-1], " \t\n\r")) + break; + ie--; + } + if(is >= ie) + return ""; + return s[is:ie]; +} + +hashname(uname: string) : string +{ + hash := 0; + prime: con 8388617; + # start after "http:" + for(i := 5; i < len uname; i++) { + hash = hash % prime; + hash = (hash << 7) + uname[i]; + } + return sys->sprint(cachedir + "/%.8ux", hash); +} diff --git a/appl/svc/webget/image2enc.b b/appl/svc/webget/image2enc.b new file mode 100644 index 00000000..8345bd40 --- /dev/null +++ b/appl/svc/webget/image2enc.b @@ -0,0 +1,1070 @@ +implement Image2enc; + +include "sys.m"; + sys: Sys; + +include "draw.m"; + +include "bufio.m"; + +include "imagefile.m"; + Rawimage: import RImagefile; + +include "image2enc.m"; + +closest:= array[16*16*16] of { + byte 255,byte 255,byte 255,byte 254,byte 254,byte 237,byte 220,byte 203, + byte 253,byte 236,byte 219,byte 202,byte 252,byte 235,byte 218,byte 201, + byte 255,byte 255,byte 255,byte 254,byte 254,byte 237,byte 220,byte 203, + byte 253,byte 236,byte 219,byte 202,byte 252,byte 235,byte 218,byte 201, + byte 255,byte 255,byte 255,byte 250,byte 250,byte 250,byte 220,byte 249, + byte 249,byte 249,byte 232,byte 248,byte 248,byte 248,byte 231,byte 201, + byte 251,byte 251,byte 250,byte 250,byte 250,byte 250,byte 249,byte 249, + byte 249,byte 249,byte 232,byte 248,byte 248,byte 248,byte 231,byte 201, + byte 251,byte 251,byte 250,byte 250,byte 250,byte 233,byte 233,byte 249, + byte 249,byte 232,byte 215,byte 215,byte 248,byte 231,byte 214,byte 197, + byte 234,byte 234,byte 250,byte 250,byte 233,byte 233,byte 216,byte 216, + byte 249,byte 232,byte 215,byte 198,byte 198,byte 231,byte 214,byte 197, + byte 217,byte 217,byte 217,byte 246,byte 233,byte 216,byte 216,byte 199, + byte 199,byte 215,byte 215,byte 198,byte 198,byte 198,byte 214,byte 197, + byte 200,byte 200,byte 246,byte 246,byte 246,byte 216,byte 199,byte 199, + byte 245,byte 245,byte 198,byte 244,byte 244,byte 244,byte 227,byte 197, + byte 247,byte 247,byte 246,byte 246,byte 246,byte 246,byte 199,byte 245, + byte 245,byte 245,byte 228,byte 244,byte 244,byte 244,byte 227,byte 193, + byte 230,byte 230,byte 246,byte 246,byte 229,byte 229,byte 212,byte 245, + byte 245,byte 228,byte 228,byte 211,byte 244,byte 227,byte 210,byte 193, + byte 213,byte 213,byte 229,byte 229,byte 212,byte 212,byte 212,byte 195, + byte 228,byte 228,byte 211,byte 211,byte 194,byte 227,byte 210,byte 193, + byte 196,byte 196,byte 242,byte 242,byte 212,byte 195,byte 195,byte 241, + byte 241,byte 211,byte 211,byte 194,byte 194,byte 240,byte 210,byte 193, + byte 243,byte 243,byte 242,byte 242,byte 242,byte 195,byte 195,byte 241, + byte 241,byte 241,byte 194,byte 194,byte 240,byte 240,byte 239,byte 205, + byte 226,byte 226,byte 242,byte 242,byte 225,byte 225,byte 195,byte 241, + byte 241,byte 224,byte 224,byte 240,byte 240,byte 239,byte 239,byte 205, + byte 209,byte 209,byte 225,byte 225,byte 208,byte 208,byte 208,byte 224, + byte 224,byte 223,byte 223,byte 223,byte 239,byte 239,byte 222,byte 205, + byte 192,byte 192,byte 192,byte 192,byte 207,byte 207,byte 207,byte 207, + byte 206,byte 206,byte 206,byte 206,byte 205,byte 205,byte 205,byte 205, + byte 255,byte 255,byte 255,byte 254,byte 254,byte 237,byte 220,byte 203, + byte 253,byte 236,byte 219,byte 202,byte 252,byte 235,byte 218,byte 201, + byte 255,byte 238,byte 221,byte 221,byte 254,byte 237,byte 220,byte 203, + byte 253,byte 236,byte 219,byte 202,byte 252,byte 235,byte 218,byte 201, + byte 255,byte 221,byte 221,byte 221,byte 204,byte 250,byte 220,byte 249, + byte 249,byte 249,byte 232,byte 248,byte 248,byte 248,byte 231,byte 201, + byte 251,byte 221,byte 221,byte 204,byte 250,byte 250,byte 249,byte 249, + byte 249,byte 249,byte 232,byte 248,byte 248,byte 248,byte 231,byte 201, + byte 251,byte 251,byte 204,byte 250,byte 250,byte 233,byte 233,byte 249, + byte 249,byte 232,byte 215,byte 215,byte 248,byte 231,byte 214,byte 197, + byte 234,byte 234,byte 250,byte 250,byte 233,byte 233,byte 216,byte 216, + byte 249,byte 232,byte 215,byte 198,byte 198,byte 231,byte 214,byte 197, + byte 217,byte 217,byte 217,byte 246,byte 233,byte 216,byte 216,byte 199, + byte 199,byte 215,byte 215,byte 198,byte 198,byte 198,byte 214,byte 197, + byte 200,byte 200,byte 246,byte 246,byte 246,byte 216,byte 199,byte 199, + byte 245,byte 245,byte 198,byte 244,byte 244,byte 244,byte 227,byte 197, + byte 247,byte 247,byte 246,byte 246,byte 246,byte 246,byte 199,byte 245, + byte 245,byte 245,byte 228,byte 244,byte 244,byte 244,byte 227,byte 193, + byte 230,byte 230,byte 246,byte 246,byte 229,byte 229,byte 212,byte 245, + byte 245,byte 228,byte 228,byte 211,byte 244,byte 227,byte 210,byte 193, + byte 213,byte 213,byte 229,byte 229,byte 212,byte 212,byte 212,byte 195, + byte 228,byte 228,byte 211,byte 211,byte 194,byte 227,byte 210,byte 193, + byte 196,byte 196,byte 242,byte 242,byte 212,byte 195,byte 195,byte 241, + byte 241,byte 211,byte 211,byte 194,byte 194,byte 240,byte 210,byte 193, + byte 243,byte 243,byte 242,byte 242,byte 242,byte 195,byte 195,byte 241, + byte 241,byte 241,byte 194,byte 194,byte 240,byte 240,byte 239,byte 205, + byte 226,byte 226,byte 242,byte 242,byte 225,byte 225,byte 195,byte 241, + byte 241,byte 224,byte 224,byte 240,byte 240,byte 239,byte 239,byte 205, + byte 209,byte 209,byte 225,byte 225,byte 208,byte 208,byte 208,byte 224, + byte 224,byte 223,byte 223,byte 223,byte 239,byte 239,byte 222,byte 205, + byte 192,byte 192,byte 192,byte 192,byte 207,byte 207,byte 207,byte 207, + byte 206,byte 206,byte 206,byte 206,byte 205,byte 205,byte 205,byte 205, + byte 255,byte 255,byte 255,byte 191,byte 191,byte 191,byte 220,byte 190, + byte 190,byte 190,byte 173,byte 189,byte 189,byte 189,byte 172,byte 201, + byte 255,byte 221,byte 221,byte 221,byte 204,byte 191,byte 220,byte 190, + byte 190,byte 190,byte 173,byte 189,byte 189,byte 189,byte 172,byte 201, + byte 255,byte 221,byte 221,byte 204,byte 204,byte 204,byte 186,byte 186, + byte 186,byte 186,byte 186,byte 185,byte 185,byte 185,byte 168,byte 201, + byte 188,byte 221,byte 204,byte 204,byte 204,byte 187,byte 186,byte 186, + byte 186,byte 186,byte 232,byte 185,byte 185,byte 185,byte 168,byte 201, + byte 188,byte 204,byte 204,byte 204,byte 187,byte 187,byte 186,byte 186, + byte 186,byte 186,byte 169,byte 185,byte 185,byte 185,byte 168,byte 197, + byte 188,byte 188,byte 204,byte 187,byte 187,byte 233,byte 216,byte 186, + byte 186,byte 186,byte 215,byte 185,byte 185,byte 185,byte 168,byte 197, + byte 217,byte 217,byte 183,byte 183,byte 183,byte 216,byte 216,byte 199, + byte 182,byte 182,byte 215,byte 198,byte 198,byte 181,byte 214,byte 197, + byte 184,byte 184,byte 183,byte 183,byte 183,byte 183,byte 199,byte 182, + byte 182,byte 182,byte 182,byte 181,byte 181,byte 181,byte 181,byte 197, + byte 184,byte 184,byte 183,byte 183,byte 183,byte 183,byte 182,byte 182, + byte 182,byte 182,byte 182,byte 181,byte 181,byte 181,byte 164,byte 193, + byte 184,byte 184,byte 183,byte 183,byte 183,byte 183,byte 182,byte 182, + byte 182,byte 228,byte 165,byte 181,byte 181,byte 164,byte 164,byte 193, + byte 167,byte 167,byte 183,byte 229,byte 166,byte 212,byte 212,byte 182, + byte 182,byte 165,byte 211,byte 211,byte 181,byte 164,byte 210,byte 193, + byte 180,byte 180,byte 179,byte 179,byte 179,byte 179,byte 195,byte 178, + byte 178,byte 178,byte 211,byte 194,byte 177,byte 177,byte 177,byte 193, + byte 180,byte 180,byte 179,byte 179,byte 179,byte 179,byte 195,byte 178, + byte 178,byte 178,byte 178,byte 177,byte 177,byte 177,byte 177,byte 205, + byte 180,byte 180,byte 179,byte 179,byte 179,byte 179,byte 178,byte 178, + byte 178,byte 161,byte 161,byte 177,byte 177,byte 177,byte 160,byte 205, + byte 163,byte 163,byte 162,byte 162,byte 162,byte 162,byte 208,byte 178, + byte 161,byte 161,byte 223,byte 177,byte 177,byte 160,byte 160,byte 205, + byte 192,byte 192,byte 192,byte 192,byte 207,byte 207,byte 207,byte 207, + byte 206,byte 206,byte 206,byte 206,byte 205,byte 205,byte 205,byte 205, + byte 176,byte 176,byte 191,byte 191,byte 191,byte 191,byte 190,byte 190, + byte 190,byte 190,byte 173,byte 189,byte 189,byte 189,byte 172,byte 201, + byte 176,byte 221,byte 221,byte 204,byte 191,byte 191,byte 190,byte 190, + byte 190,byte 190,byte 173,byte 189,byte 189,byte 189,byte 172,byte 201, + byte 188,byte 221,byte 204,byte 204,byte 204,byte 187,byte 186,byte 186, + byte 186,byte 186,byte 173,byte 185,byte 185,byte 185,byte 168,byte 201, + byte 188,byte 204,byte 204,byte 204,byte 187,byte 187,byte 186,byte 186, + byte 186,byte 186,byte 169,byte 185,byte 185,byte 185,byte 168,byte 201, + byte 188,byte 188,byte 204,byte 187,byte 187,byte 187,byte 186,byte 186, + byte 186,byte 186,byte 169,byte 185,byte 185,byte 185,byte 168,byte 197, + byte 188,byte 188,byte 187,byte 187,byte 187,byte 170,byte 170,byte 186, + byte 186,byte 169,byte 169,byte 185,byte 185,byte 168,byte 168,byte 197, + byte 184,byte 184,byte 183,byte 183,byte 183,byte 170,byte 170,byte 182, + byte 182,byte 169,byte 152,byte 152,byte 181,byte 168,byte 151,byte 197, + byte 184,byte 184,byte 183,byte 183,byte 183,byte 183,byte 182,byte 182, + byte 182,byte 182,byte 182,byte 181,byte 181,byte 181,byte 164,byte 197, + byte 184,byte 184,byte 183,byte 183,byte 183,byte 183,byte 182,byte 182, + byte 182,byte 182,byte 165,byte 181,byte 181,byte 181,byte 164,byte 193, + byte 184,byte 184,byte 183,byte 183,byte 183,byte 166,byte 166,byte 182, + byte 182,byte 165,byte 165,byte 181,byte 181,byte 164,byte 164,byte 193, + byte 167,byte 167,byte 167,byte 166,byte 166,byte 166,byte 149,byte 182, + byte 165,byte 165,byte 165,byte 148,byte 181,byte 164,byte 147,byte 193, + byte 180,byte 180,byte 179,byte 179,byte 179,byte 179,byte 149,byte 178, + byte 178,byte 178,byte 148,byte 177,byte 177,byte 177,byte 147,byte 193, + byte 180,byte 180,byte 179,byte 179,byte 179,byte 179,byte 178,byte 178, + byte 178,byte 178,byte 178,byte 177,byte 177,byte 177,byte 160,byte 205, + byte 180,byte 180,byte 179,byte 179,byte 179,byte 162,byte 162,byte 178, + byte 178,byte 161,byte 161,byte 177,byte 177,byte 160,byte 160,byte 205, + byte 163,byte 163,byte 162,byte 162,byte 162,byte 162,byte 145,byte 161, + byte 161,byte 161,byte 144,byte 144,byte 160,byte 160,byte 160,byte 205, + byte 192,byte 192,byte 192,byte 192,byte 207,byte 207,byte 207,byte 207, + byte 206,byte 206,byte 206,byte 206,byte 205,byte 205,byte 205,byte 205, + byte 176,byte 176,byte 191,byte 191,byte 191,byte 174,byte 174,byte 190, + byte 190,byte 173,byte 156,byte 156,byte 189,byte 172,byte 155,byte 138, + byte 176,byte 176,byte 204,byte 191,byte 191,byte 174,byte 174,byte 190, + byte 190,byte 173,byte 156,byte 156,byte 189,byte 172,byte 155,byte 138, + byte 188,byte 204,byte 204,byte 204,byte 187,byte 187,byte 186,byte 186, + byte 186,byte 186,byte 169,byte 185,byte 185,byte 185,byte 168,byte 138, + byte 188,byte 188,byte 204,byte 187,byte 187,byte 187,byte 186,byte 186, + byte 186,byte 186,byte 169,byte 185,byte 185,byte 185,byte 168,byte 138, + byte 188,byte 188,byte 187,byte 187,byte 187,byte 170,byte 170,byte 186, + byte 186,byte 169,byte 169,byte 185,byte 185,byte 168,byte 151,byte 134, + byte 171,byte 171,byte 187,byte 187,byte 170,byte 170,byte 170,byte 186, + byte 186,byte 169,byte 152,byte 152,byte 185,byte 168,byte 151,byte 134, + byte 171,byte 171,byte 183,byte 183,byte 170,byte 170,byte 170,byte 153, + byte 182,byte 169,byte 152,byte 135,byte 135,byte 168,byte 151,byte 134, + byte 184,byte 184,byte 183,byte 183,byte 183,byte 183,byte 153,byte 182, + byte 182,byte 182,byte 182,byte 181,byte 181,byte 181,byte 164,byte 134, + byte 184,byte 184,byte 183,byte 183,byte 183,byte 183,byte 182,byte 182, + byte 182,byte 182,byte 165,byte 181,byte 181,byte 181,byte 164,byte 130, + byte 167,byte 167,byte 183,byte 183,byte 166,byte 166,byte 166,byte 182, + byte 182,byte 165,byte 165,byte 181,byte 181,byte 164,byte 147,byte 130, + byte 150,byte 150,byte 166,byte 166,byte 166,byte 149,byte 149,byte 182, + byte 165,byte 165,byte 148,byte 148,byte 164,byte 164,byte 147,byte 130, + byte 150,byte 150,byte 179,byte 179,byte 179,byte 149,byte 132,byte 178, + byte 178,byte 178,byte 148,byte 131,byte 177,byte 177,byte 147,byte 130, + byte 180,byte 180,byte 179,byte 179,byte 179,byte 179,byte 132,byte 178, + byte 178,byte 178,byte 161,byte 177,byte 177,byte 177,byte 160,byte 142, + byte 163,byte 163,byte 179,byte 179,byte 162,byte 162,byte 162,byte 178, + byte 178,byte 161,byte 161,byte 177,byte 177,byte 160,byte 160,byte 142, + byte 146,byte 146,byte 162,byte 162,byte 145,byte 145,byte 145,byte 161, + byte 161,byte 144,byte 144,byte 144,byte 160,byte 160,byte 159,byte 142, + byte 129,byte 129,byte 129,byte 129,byte 128,byte 128,byte 128,byte 128, + byte 143,byte 143,byte 143,byte 143,byte 142,byte 142,byte 142,byte 142, + byte 175,byte 175,byte 191,byte 191,byte 174,byte 174,byte 157,byte 157, + byte 190,byte 173,byte 156,byte 139,byte 139,byte 172,byte 155,byte 138, + byte 175,byte 175,byte 191,byte 191,byte 174,byte 174,byte 157,byte 157, + byte 190,byte 173,byte 156,byte 139,byte 139,byte 172,byte 155,byte 138, + byte 188,byte 188,byte 204,byte 187,byte 187,byte 187,byte 157,byte 186, + byte 186,byte 186,byte 156,byte 185,byte 185,byte 185,byte 168,byte 138, + byte 188,byte 188,byte 187,byte 187,byte 187,byte 170,byte 170,byte 186, + byte 186,byte 169,byte 169,byte 185,byte 185,byte 168,byte 168,byte 138, + byte 171,byte 171,byte 187,byte 187,byte 170,byte 170,byte 170,byte 186, + byte 186,byte 169,byte 152,byte 152,byte 185,byte 168,byte 151,byte 134, + byte 171,byte 171,byte 187,byte 170,byte 170,byte 170,byte 170,byte 153, + byte 169,byte 169,byte 152,byte 135,byte 135,byte 168,byte 151,byte 134, + byte 154,byte 154,byte 154,byte 170,byte 170,byte 170,byte 153,byte 153, + byte 169,byte 152,byte 152,byte 135,byte 135,byte 135,byte 151,byte 134, + byte 154,byte 154,byte 183,byte 183,byte 183,byte 153,byte 153,byte 153, + byte 182,byte 182,byte 135,byte 135,byte 181,byte 181,byte 164,byte 134, + byte 184,byte 184,byte 183,byte 183,byte 183,byte 166,byte 166,byte 182, + byte 182,byte 165,byte 165,byte 181,byte 181,byte 164,byte 164,byte 130, + byte 167,byte 167,byte 183,byte 166,byte 166,byte 166,byte 149,byte 182, + byte 165,byte 165,byte 165,byte 148,byte 181,byte 164,byte 147,byte 130, + byte 150,byte 150,byte 150,byte 166,byte 149,byte 149,byte 149,byte 132, + byte 165,byte 165,byte 148,byte 148,byte 131,byte 147,byte 147,byte 130, + byte 133,byte 133,byte 179,byte 179,byte 149,byte 132,byte 132,byte 132, + byte 178,byte 148,byte 148,byte 131,byte 131,byte 131,byte 130,byte 130, + byte 133,byte 133,byte 179,byte 179,byte 179,byte 132,byte 132,byte 178, + byte 178,byte 178,byte 131,byte 131,byte 131,byte 177,byte 160,byte 142, + byte 163,byte 163,byte 179,byte 162,byte 162,byte 162,byte 132,byte 178, + byte 161,byte 161,byte 144,byte 131,byte 177,byte 160,byte 160,byte 142, + byte 146,byte 146,byte 162,byte 162,byte 145,byte 145,byte 145,byte 161, + byte 161,byte 144,byte 144,byte 143,byte 160,byte 160,byte 159,byte 142, + byte 129,byte 129,byte 129,byte 129,byte 128,byte 128,byte 128,byte 128, + byte 143,byte 143,byte 143,byte 143,byte 142,byte 142,byte 142,byte 142, + byte 158,byte 158,byte 158,byte 112,byte 174,byte 157,byte 157,byte 140, + byte 140,byte 156,byte 156,byte 139,byte 139,byte 139,byte 155,byte 138, + byte 158,byte 158,byte 158,byte 112,byte 174,byte 157,byte 157,byte 140, + byte 140,byte 156,byte 156,byte 139,byte 139,byte 139,byte 155,byte 138, + byte 158,byte 158,byte 124,byte 124,byte 124,byte 157,byte 157,byte 140, + byte 123,byte 123,byte 156,byte 139,byte 139,byte 122,byte 155,byte 138, + byte 125,byte 125,byte 124,byte 124,byte 124,byte 170,byte 170,byte 123, + byte 123,byte 169,byte 152,byte 152,byte 122,byte 168,byte 151,byte 138, + byte 171,byte 171,byte 124,byte 124,byte 170,byte 170,byte 170,byte 153, + byte 123,byte 169,byte 152,byte 135,byte 135,byte 168,byte 151,byte 134, + byte 154,byte 154,byte 154,byte 170,byte 170,byte 170,byte 153,byte 153, + byte 169,byte 152,byte 152,byte 135,byte 135,byte 135,byte 151,byte 134, + byte 154,byte 154,byte 154,byte 170,byte 170,byte 153,byte 153,byte 153, + byte 136,byte 152,byte 135,byte 135,byte 135,byte 135,byte 134,byte 134, + byte 137,byte 137,byte 137,byte 120,byte 153,byte 153,byte 153,byte 136, + byte 136,byte 136,byte 135,byte 135,byte 135,byte 118,byte 164,byte 134, + byte 137,byte 137,byte 120,byte 120,byte 120,byte 166,byte 136,byte 136, + byte 136,byte 165,byte 165,byte 118,byte 118,byte 164,byte 147,byte 130, + byte 150,byte 150,byte 120,byte 166,byte 166,byte 149,byte 149,byte 136, + byte 165,byte 165,byte 148,byte 148,byte 118,byte 164,byte 147,byte 130, + byte 150,byte 150,byte 150,byte 149,byte 149,byte 149,byte 132,byte 132, + byte 165,byte 148,byte 148,byte 131,byte 131,byte 147,byte 147,byte 130, + byte 133,byte 133,byte 133,byte 149,byte 132,byte 132,byte 132,byte 132, + byte 115,byte 148,byte 131,byte 131,byte 131,byte 131,byte 130,byte 130, + byte 133,byte 133,byte 133,byte 116,byte 132,byte 132,byte 132,byte 132, + byte 115,byte 115,byte 131,byte 131,byte 131,byte 131,byte 160,byte 142, + byte 133,byte 133,byte 116,byte 162,byte 162,byte 132,byte 132,byte 115, + byte 161,byte 161,byte 144,byte 131,byte 131,byte 160,byte 160,byte 142, + byte 146,byte 146,byte 146,byte 145,byte 145,byte 145,byte 128,byte 161, + byte 144,byte 144,byte 144,byte 143,byte 160,byte 160,byte 159,byte 142, + byte 129,byte 129,byte 129,byte 129,byte 128,byte 128,byte 128,byte 128, + byte 143,byte 143,byte 143,byte 143,byte 142,byte 142,byte 142,byte 142, + byte 141,byte 141,byte 112,byte 112,byte 112,byte 157,byte 140,byte 140, + byte 140,byte 127,byte 139,byte 126,byte 126,byte 126,byte 109,byte 138, + byte 141,byte 141,byte 112,byte 112,byte 112,byte 157,byte 140,byte 140, + byte 140,byte 127,byte 139,byte 126,byte 126,byte 126,byte 109,byte 138, + byte 125,byte 125,byte 124,byte 124,byte 124,byte 124,byte 140,byte 123, + byte 123,byte 123,byte 123,byte 122,byte 122,byte 122,byte 122,byte 138, + byte 125,byte 125,byte 124,byte 124,byte 124,byte 124,byte 123,byte 123, + byte 123,byte 123,byte 123,byte 122,byte 122,byte 122,byte 105,byte 138, + byte 125,byte 125,byte 124,byte 124,byte 124,byte 124,byte 153,byte 123, + byte 123,byte 123,byte 152,byte 122,byte 122,byte 122,byte 105,byte 134, + byte 154,byte 154,byte 124,byte 124,byte 124,byte 153,byte 153,byte 153, + byte 123,byte 123,byte 135,byte 135,byte 122,byte 122,byte 105,byte 134, + byte 137,byte 137,byte 137,byte 120,byte 153,byte 153,byte 153,byte 136, + byte 136,byte 136,byte 135,byte 135,byte 135,byte 118,byte 105,byte 134, + byte 137,byte 137,byte 120,byte 120,byte 120,byte 153,byte 136,byte 136, + byte 136,byte 119,byte 119,byte 118,byte 118,byte 118,byte 118,byte 134, + byte 137,byte 137,byte 120,byte 120,byte 120,byte 120,byte 136,byte 136, + byte 119,byte 119,byte 119,byte 118,byte 118,byte 118,byte 101,byte 130, + byte 121,byte 121,byte 120,byte 120,byte 120,byte 120,byte 136,byte 119, + byte 119,byte 119,byte 102,byte 118,byte 118,byte 118,byte 101,byte 130, + byte 133,byte 133,byte 120,byte 120,byte 149,byte 132,byte 132,byte 119, + byte 119,byte 102,byte 148,byte 131,byte 131,byte 101,byte 101,byte 130, + byte 117,byte 117,byte 116,byte 116,byte 116,byte 132,byte 132,byte 115, + byte 115,byte 115,byte 131,byte 131,byte 114,byte 114,byte 114,byte 130, + byte 117,byte 117,byte 116,byte 116,byte 116,byte 116,byte 132,byte 115, + byte 115,byte 115,byte 131,byte 114,byte 114,byte 114,byte 114,byte 142, + byte 117,byte 117,byte 116,byte 116,byte 116,byte 116,byte 115,byte 115, + byte 115,byte 115,byte 98,byte 114,byte 114,byte 114,byte 97,byte 142, + byte 100,byte 100,byte 116,byte 99,byte 99,byte 99,byte 99,byte 115, + byte 98,byte 98,byte 98,byte 114,byte 114,byte 97,byte 97,byte 142, + byte 129,byte 129,byte 129,byte 129,byte 128,byte 128,byte 128,byte 128, + byte 143,byte 143,byte 143,byte 143,byte 142,byte 142,byte 142,byte 142, + byte 113,byte 113,byte 112,byte 112,byte 112,byte 112,byte 140,byte 140, + byte 127,byte 127,byte 110,byte 126,byte 126,byte 126,byte 109,byte 75, + byte 113,byte 113,byte 112,byte 112,byte 112,byte 112,byte 140,byte 140, + byte 127,byte 127,byte 110,byte 126,byte 126,byte 126,byte 109,byte 75, + byte 125,byte 125,byte 124,byte 124,byte 124,byte 124,byte 123,byte 123, + byte 123,byte 123,byte 123,byte 122,byte 122,byte 122,byte 105,byte 75, + byte 125,byte 125,byte 124,byte 124,byte 124,byte 124,byte 123,byte 123, + byte 123,byte 123,byte 106,byte 122,byte 122,byte 122,byte 105,byte 75, + byte 125,byte 125,byte 124,byte 124,byte 124,byte 124,byte 123,byte 123, + byte 123,byte 123,byte 106,byte 122,byte 122,byte 122,byte 105,byte 71, + byte 125,byte 125,byte 124,byte 124,byte 124,byte 107,byte 107,byte 123, + byte 123,byte 106,byte 106,byte 122,byte 122,byte 105,byte 105,byte 71, + byte 137,byte 137,byte 120,byte 120,byte 120,byte 107,byte 136,byte 136, + byte 136,byte 106,byte 106,byte 118,byte 118,byte 105,byte 88,byte 71, + byte 137,byte 137,byte 120,byte 120,byte 120,byte 120,byte 136,byte 136, + byte 119,byte 119,byte 119,byte 118,byte 118,byte 118,byte 101,byte 71, + byte 121,byte 121,byte 120,byte 120,byte 120,byte 120,byte 136,byte 119, + byte 119,byte 119,byte 102,byte 118,byte 118,byte 118,byte 101,byte 67, + byte 121,byte 121,byte 120,byte 120,byte 120,byte 103,byte 103,byte 119, + byte 119,byte 102,byte 102,byte 118,byte 118,byte 101,byte 101,byte 67, + byte 104,byte 104,byte 120,byte 103,byte 103,byte 103,byte 103,byte 119, + byte 102,byte 102,byte 102,byte 118,byte 118,byte 101,byte 84,byte 67, + byte 117,byte 117,byte 116,byte 116,byte 116,byte 116,byte 115,byte 115, + byte 115,byte 115,byte 115,byte 114,byte 114,byte 114,byte 114,byte 67, + byte 117,byte 117,byte 116,byte 116,byte 116,byte 116,byte 115,byte 115, + byte 115,byte 115,byte 115,byte 114,byte 114,byte 114,byte 97,byte 79, + byte 117,byte 117,byte 116,byte 116,byte 116,byte 99,byte 99,byte 115, + byte 115,byte 98,byte 98,byte 114,byte 114,byte 97,byte 97,byte 79, + byte 100,byte 100,byte 99,byte 99,byte 99,byte 99,byte 82,byte 98, + byte 98,byte 98,byte 81,byte 114,byte 97,byte 97,byte 97,byte 79, + byte 66,byte 66,byte 66,byte 66,byte 65,byte 65,byte 65,byte 65, + byte 64,byte 64,byte 64,byte 64,byte 79,byte 79,byte 79,byte 79, + byte 96,byte 96,byte 112,byte 112,byte 111,byte 111,byte 94,byte 127, + byte 127,byte 110,byte 110,byte 93,byte 126,byte 109,byte 92,byte 75, + byte 96,byte 96,byte 112,byte 112,byte 111,byte 111,byte 94,byte 127, + byte 127,byte 110,byte 110,byte 93,byte 126,byte 109,byte 92,byte 75, + byte 125,byte 125,byte 124,byte 124,byte 124,byte 124,byte 123,byte 123, + byte 123,byte 123,byte 106,byte 122,byte 122,byte 105,byte 105,byte 75, + byte 125,byte 125,byte 124,byte 124,byte 124,byte 107,byte 107,byte 123, + byte 123,byte 106,byte 106,byte 122,byte 122,byte 105,byte 105,byte 75, + byte 108,byte 108,byte 124,byte 124,byte 107,byte 107,byte 107,byte 123, + byte 123,byte 106,byte 106,byte 122,byte 122,byte 105,byte 88,byte 71, + byte 108,byte 108,byte 124,byte 107,byte 107,byte 107,byte 90,byte 123, + byte 106,byte 106,byte 106,byte 89,byte 122,byte 105,byte 88,byte 71, + byte 91,byte 91,byte 120,byte 107,byte 107,byte 90,byte 90,byte 136, + byte 106,byte 106,byte 89,byte 89,byte 118,byte 105,byte 88,byte 71, + byte 121,byte 121,byte 120,byte 120,byte 120,byte 120,byte 136,byte 119, + byte 119,byte 119,byte 102,byte 118,byte 118,byte 118,byte 101,byte 71, + byte 121,byte 121,byte 120,byte 120,byte 120,byte 103,byte 103,byte 119, + byte 119,byte 102,byte 102,byte 118,byte 118,byte 101,byte 101,byte 67, + byte 104,byte 104,byte 120,byte 103,byte 103,byte 103,byte 103,byte 119, + byte 102,byte 102,byte 102,byte 118,byte 118,byte 101,byte 84,byte 67, + byte 104,byte 104,byte 103,byte 103,byte 103,byte 103,byte 86,byte 102, + byte 102,byte 102,byte 85,byte 85,byte 101,byte 101,byte 84,byte 67, + byte 87,byte 87,byte 116,byte 116,byte 116,byte 86,byte 86,byte 115, + byte 115,byte 115,byte 85,byte 85,byte 114,byte 114,byte 84,byte 67, + byte 117,byte 117,byte 116,byte 116,byte 116,byte 116,byte 115,byte 115, + byte 115,byte 115,byte 98,byte 114,byte 114,byte 114,byte 97,byte 79, + byte 100,byte 100,byte 99,byte 99,byte 99,byte 99,byte 99,byte 115, + byte 98,byte 98,byte 98,byte 114,byte 114,byte 97,byte 97,byte 79, + byte 83,byte 83,byte 99,byte 99,byte 82,byte 82,byte 82,byte 98, + byte 98,byte 81,byte 81,byte 81,byte 97,byte 97,byte 80,byte 79, + byte 66,byte 66,byte 66,byte 66,byte 65,byte 65,byte 65,byte 65, + byte 64,byte 64,byte 64,byte 64,byte 79,byte 79,byte 79,byte 79, + byte 95,byte 95,byte 111,byte 111,byte 94,byte 94,byte 94,byte 77, + byte 110,byte 110,byte 93,byte 93,byte 76,byte 109,byte 92,byte 75, + byte 95,byte 95,byte 111,byte 111,byte 94,byte 94,byte 94,byte 77, + byte 110,byte 110,byte 93,byte 93,byte 76,byte 109,byte 92,byte 75, + byte 108,byte 108,byte 124,byte 111,byte 107,byte 94,byte 94,byte 123, + byte 123,byte 106,byte 93,byte 93,byte 122,byte 105,byte 92,byte 75, + byte 108,byte 108,byte 108,byte 107,byte 107,byte 107,byte 90,byte 123, + byte 106,byte 106,byte 106,byte 89,byte 122,byte 105,byte 88,byte 75, + byte 91,byte 91,byte 107,byte 107,byte 107,byte 90,byte 90,byte 123, + byte 106,byte 106,byte 89,byte 89,byte 105,byte 105,byte 88,byte 71, + byte 91,byte 91,byte 91,byte 107,byte 90,byte 90,byte 90,byte 73, + byte 106,byte 106,byte 89,byte 89,byte 72,byte 88,byte 88,byte 71, + byte 91,byte 91,byte 91,byte 90,byte 90,byte 90,byte 73,byte 73, + byte 106,byte 89,byte 89,byte 72,byte 72,byte 88,byte 88,byte 71, + byte 74,byte 74,byte 120,byte 120,byte 120,byte 73,byte 73,byte 119, + byte 119,byte 102,byte 89,byte 72,byte 72,byte 101,byte 101,byte 71, + byte 104,byte 104,byte 120,byte 103,byte 103,byte 103,byte 103,byte 119, + byte 102,byte 102,byte 102,byte 118,byte 118,byte 101,byte 84,byte 67, + byte 104,byte 104,byte 103,byte 103,byte 103,byte 103,byte 86,byte 102, + byte 102,byte 102,byte 85,byte 85,byte 101,byte 101,byte 84,byte 67, + byte 87,byte 87,byte 87,byte 103,byte 86,byte 86,byte 86,byte 86, + byte 102,byte 85,byte 85,byte 85,byte 85,byte 84,byte 84,byte 67, + byte 87,byte 87,byte 87,byte 86,byte 86,byte 86,byte 69,byte 69, + byte 115,byte 85,byte 85,byte 85,byte 68,byte 68,byte 67,byte 67, + byte 70,byte 70,byte 116,byte 116,byte 99,byte 69,byte 69,byte 69, + byte 115,byte 98,byte 85,byte 68,byte 68,byte 97,byte 97,byte 79, + byte 100,byte 100,byte 99,byte 99,byte 99,byte 82,byte 82,byte 98, + byte 98,byte 98,byte 81,byte 68,byte 97,byte 97,byte 97,byte 79, + byte 83,byte 83,byte 83,byte 82,byte 82,byte 82,byte 82,byte 98, + byte 81,byte 81,byte 81,byte 64,byte 97,byte 97,byte 80,byte 79, + byte 66,byte 66,byte 66,byte 66,byte 65,byte 65,byte 65,byte 65, + byte 64,byte 64,byte 64,byte 64,byte 79,byte 79,byte 79,byte 79, + byte 78,byte 78,byte 49,byte 49,byte 94,byte 77,byte 77,byte 48, + byte 48,byte 93,byte 93,byte 76,byte 76,byte 63,byte 92,byte 75, + byte 78,byte 78,byte 49,byte 49,byte 94,byte 77,byte 77,byte 48, + byte 48,byte 93,byte 93,byte 76,byte 76,byte 63,byte 92,byte 75, + byte 62,byte 62,byte 61,byte 61,byte 61,byte 61,byte 77,byte 60, + byte 60,byte 60,byte 93,byte 76,byte 59,byte 59,byte 59,byte 75, + byte 62,byte 62,byte 61,byte 61,byte 61,byte 61,byte 90,byte 60, + byte 60,byte 60,byte 89,byte 59,byte 59,byte 59,byte 88,byte 75, + byte 91,byte 91,byte 61,byte 61,byte 61,byte 90,byte 73,byte 60, + byte 60,byte 60,byte 89,byte 72,byte 59,byte 59,byte 88,byte 71, + byte 74,byte 74,byte 61,byte 61,byte 90,byte 73,byte 73,byte 73, + byte 60,byte 89,byte 89,byte 72,byte 72,byte 72,byte 71,byte 71, + byte 74,byte 74,byte 74,byte 90,byte 73,byte 73,byte 73,byte 73, + byte 56,byte 89,byte 72,byte 72,byte 72,byte 72,byte 71,byte 71, + byte 58,byte 58,byte 57,byte 57,byte 57,byte 73,byte 73,byte 56, + byte 56,byte 56,byte 72,byte 72,byte 55,byte 55,byte 55,byte 71, + byte 58,byte 58,byte 57,byte 57,byte 57,byte 57,byte 56,byte 56, + byte 56,byte 56,byte 56,byte 55,byte 55,byte 55,byte 55,byte 67, + byte 87,byte 87,byte 57,byte 57,byte 57,byte 86,byte 86,byte 56, + byte 56,byte 56,byte 85,byte 85,byte 55,byte 55,byte 84,byte 67, + byte 87,byte 87,byte 87,byte 86,byte 86,byte 86,byte 69,byte 69, + byte 56,byte 85,byte 85,byte 85,byte 68,byte 68,byte 67,byte 67, + byte 70,byte 70,byte 70,byte 53,byte 69,byte 69,byte 69,byte 69, + byte 52,byte 85,byte 85,byte 68,byte 68,byte 68,byte 67,byte 67, + byte 70,byte 70,byte 53,byte 53,byte 53,byte 69,byte 69,byte 52, + byte 52,byte 52,byte 68,byte 68,byte 68,byte 51,byte 51,byte 79, + byte 54,byte 54,byte 53,byte 53,byte 53,byte 69,byte 69,byte 52, + byte 52,byte 52,byte 68,byte 68,byte 51,byte 51,byte 80,byte 79, + byte 83,byte 83,byte 53,byte 82,byte 82,byte 65,byte 65,byte 52, + byte 52,byte 81,byte 64,byte 64,byte 51,byte 80,byte 80,byte 79, + byte 66,byte 66,byte 66,byte 66,byte 65,byte 65,byte 65,byte 65, + byte 64,byte 64,byte 64,byte 64,byte 79,byte 79,byte 79,byte 79, + byte 50,byte 50,byte 49,byte 49,byte 49,byte 77,byte 77,byte 48, + byte 48,byte 48,byte 76,byte 76,byte 63,byte 63,byte 46,byte 12, + byte 50,byte 50,byte 49,byte 49,byte 49,byte 77,byte 77,byte 48, + byte 48,byte 48,byte 76,byte 76,byte 63,byte 63,byte 46,byte 12, + byte 62,byte 62,byte 61,byte 61,byte 61,byte 61,byte 77,byte 60, + byte 60,byte 60,byte 60,byte 59,byte 59,byte 59,byte 59,byte 12, + byte 62,byte 62,byte 61,byte 61,byte 61,byte 61,byte 60,byte 60, + byte 60,byte 60,byte 60,byte 59,byte 59,byte 59,byte 42,byte 12, + byte 62,byte 62,byte 61,byte 61,byte 61,byte 61,byte 73,byte 60, + byte 60,byte 60,byte 43,byte 59,byte 59,byte 59,byte 42,byte 8, + byte 74,byte 74,byte 61,byte 61,byte 61,byte 73,byte 73,byte 60, + byte 60,byte 60,byte 72,byte 72,byte 72,byte 59,byte 42,byte 8, + byte 74,byte 74,byte 74,byte 57,byte 73,byte 73,byte 73,byte 73, + byte 56,byte 56,byte 72,byte 72,byte 72,byte 72,byte 42,byte 8, + byte 58,byte 58,byte 57,byte 57,byte 57,byte 57,byte 73,byte 56, + byte 56,byte 56,byte 72,byte 55,byte 55,byte 55,byte 55,byte 8, + byte 58,byte 58,byte 57,byte 57,byte 57,byte 57,byte 56,byte 56, + byte 56,byte 56,byte 56,byte 55,byte 55,byte 55,byte 38,byte 4, + byte 58,byte 58,byte 57,byte 57,byte 57,byte 57,byte 56,byte 56, + byte 56,byte 56,byte 39,byte 55,byte 55,byte 55,byte 38,byte 4, + byte 70,byte 70,byte 57,byte 57,byte 40,byte 69,byte 69,byte 69, + byte 56,byte 39,byte 85,byte 68,byte 68,byte 38,byte 38,byte 4, + byte 70,byte 70,byte 53,byte 53,byte 53,byte 69,byte 69,byte 52, + byte 52,byte 52,byte 68,byte 68,byte 68,byte 51,byte 51,byte 4, + byte 54,byte 54,byte 53,byte 53,byte 53,byte 69,byte 69,byte 52, + byte 52,byte 52,byte 68,byte 68,byte 51,byte 51,byte 51,byte 0, + byte 54,byte 54,byte 53,byte 53,byte 53,byte 53,byte 69,byte 52, + byte 52,byte 52,byte 35,byte 51,byte 51,byte 51,byte 34,byte 0, + byte 37,byte 37,byte 53,byte 36,byte 36,byte 36,byte 36,byte 52, + byte 35,byte 35,byte 35,byte 51,byte 51,byte 34,byte 34,byte 0, + byte 3,byte 3,byte 3,byte 3,byte 2,byte 2,byte 2,byte 2, + byte 1,byte 1,byte 1,byte 1,byte 0,byte 0,byte 0,byte 0, + byte 33,byte 33,byte 49,byte 49,byte 32,byte 32,byte 77,byte 48, + byte 48,byte 47,byte 47,byte 63,byte 63,byte 46,byte 46,byte 12, + byte 33,byte 33,byte 49,byte 49,byte 32,byte 32,byte 77,byte 48, + byte 48,byte 47,byte 47,byte 63,byte 63,byte 46,byte 46,byte 12, + byte 62,byte 62,byte 61,byte 61,byte 61,byte 61,byte 60,byte 60, + byte 60,byte 43,byte 43,byte 59,byte 59,byte 59,byte 42,byte 12, + byte 62,byte 62,byte 61,byte 61,byte 61,byte 44,byte 44,byte 60, + byte 60,byte 43,byte 43,byte 59,byte 59,byte 42,byte 42,byte 12, + byte 45,byte 45,byte 61,byte 61,byte 44,byte 44,byte 44,byte 60, + byte 60,byte 43,byte 43,byte 59,byte 59,byte 42,byte 42,byte 8, + byte 45,byte 45,byte 61,byte 44,byte 44,byte 44,byte 73,byte 60, + byte 43,byte 43,byte 26,byte 72,byte 59,byte 42,byte 42,byte 8, + byte 74,byte 74,byte 57,byte 44,byte 44,byte 73,byte 73,byte 56, + byte 43,byte 43,byte 26,byte 72,byte 72,byte 42,byte 42,byte 8, + byte 58,byte 58,byte 57,byte 57,byte 57,byte 57,byte 56,byte 56, + byte 56,byte 56,byte 39,byte 55,byte 55,byte 55,byte 38,byte 8, + byte 58,byte 58,byte 57,byte 57,byte 57,byte 40,byte 40,byte 56, + byte 56,byte 39,byte 39,byte 55,byte 55,byte 38,byte 38,byte 4, + byte 41,byte 41,byte 40,byte 40,byte 40,byte 40,byte 40,byte 56, + byte 39,byte 39,byte 39,byte 55,byte 55,byte 38,byte 38,byte 4, + byte 41,byte 41,byte 40,byte 40,byte 40,byte 23,byte 23,byte 39, + byte 39,byte 39,byte 22,byte 68,byte 38,byte 38,byte 38,byte 4, + byte 54,byte 54,byte 53,byte 53,byte 53,byte 69,byte 69,byte 52, + byte 52,byte 52,byte 68,byte 68,byte 51,byte 51,byte 21,byte 4, + byte 54,byte 54,byte 53,byte 53,byte 53,byte 53,byte 69,byte 52, + byte 52,byte 52,byte 35,byte 51,byte 51,byte 51,byte 34,byte 0, + byte 37,byte 37,byte 53,byte 36,byte 36,byte 36,byte 36,byte 52, + byte 35,byte 35,byte 35,byte 51,byte 51,byte 34,byte 34,byte 0, + byte 37,byte 37,byte 36,byte 36,byte 36,byte 36,byte 36,byte 35, + byte 35,byte 35,byte 35,byte 18,byte 34,byte 34,byte 34,byte 0, + byte 3,byte 3,byte 3,byte 3,byte 2,byte 2,byte 2,byte 2, + byte 1,byte 1,byte 1,byte 1,byte 0,byte 0,byte 0,byte 0, + byte 16,byte 16,byte 32,byte 32,byte 31,byte 31,byte 31,byte 47, + byte 47,byte 30,byte 30,byte 30,byte 46,byte 46,byte 29,byte 12, + byte 16,byte 16,byte 32,byte 32,byte 31,byte 31,byte 31,byte 47, + byte 47,byte 30,byte 30,byte 30,byte 46,byte 46,byte 29,byte 12, + byte 45,byte 45,byte 44,byte 44,byte 44,byte 44,byte 31,byte 60, + byte 43,byte 43,byte 30,byte 59,byte 59,byte 42,byte 42,byte 12, + byte 45,byte 45,byte 44,byte 44,byte 44,byte 44,byte 27,byte 43, + byte 43,byte 43,byte 26,byte 26,byte 42,byte 42,byte 42,byte 12, + byte 28,byte 28,byte 44,byte 44,byte 27,byte 27,byte 27,byte 43, + byte 43,byte 26,byte 26,byte 26,byte 42,byte 42,byte 25,byte 8, + byte 28,byte 28,byte 44,byte 44,byte 27,byte 27,byte 27,byte 43, + byte 43,byte 26,byte 26,byte 9,byte 42,byte 42,byte 25,byte 8, + byte 28,byte 28,byte 28,byte 27,byte 27,byte 27,byte 10,byte 43, + byte 26,byte 26,byte 26,byte 9,byte 42,byte 42,byte 25,byte 8, + byte 41,byte 41,byte 57,byte 40,byte 40,byte 40,byte 40,byte 56, + byte 39,byte 39,byte 39,byte 55,byte 55,byte 38,byte 38,byte 8, + byte 41,byte 41,byte 40,byte 40,byte 40,byte 40,byte 23,byte 39, + byte 39,byte 39,byte 22,byte 55,byte 38,byte 38,byte 38,byte 4, + byte 24,byte 24,byte 40,byte 40,byte 23,byte 23,byte 23,byte 39, + byte 39,byte 22,byte 22,byte 22,byte 38,byte 38,byte 21,byte 4, + byte 24,byte 24,byte 24,byte 23,byte 23,byte 23,byte 23,byte 39, + byte 22,byte 22,byte 22,byte 5,byte 38,byte 38,byte 21,byte 4, + byte 24,byte 24,byte 53,byte 23,byte 23,byte 6,byte 6,byte 52, + byte 52,byte 22,byte 5,byte 5,byte 51,byte 21,byte 21,byte 4, + byte 37,byte 37,byte 53,byte 36,byte 36,byte 36,byte 36,byte 52, + byte 35,byte 35,byte 35,byte 51,byte 51,byte 34,byte 34,byte 0, + byte 37,byte 37,byte 36,byte 36,byte 36,byte 36,byte 36,byte 35, + byte 35,byte 35,byte 35,byte 18,byte 34,byte 34,byte 34,byte 0, + byte 20,byte 20,byte 36,byte 36,byte 19,byte 19,byte 19,byte 35, + byte 35,byte 18,byte 18,byte 18,byte 34,byte 34,byte 17,byte 0, + byte 3,byte 3,byte 3,byte 3,byte 2,byte 2,byte 2,byte 2, + byte 1,byte 1,byte 1,byte 1,byte 0,byte 0,byte 0,byte 0, + byte 15,byte 15,byte 15,byte 15,byte 14,byte 14,byte 14,byte 14, + byte 13,byte 13,byte 13,byte 13,byte 12,byte 12,byte 12,byte 12, + byte 15,byte 15,byte 15,byte 15,byte 14,byte 14,byte 14,byte 14, + byte 13,byte 13,byte 13,byte 13,byte 12,byte 12,byte 12,byte 12, + byte 15,byte 15,byte 15,byte 15,byte 14,byte 14,byte 14,byte 14, + byte 13,byte 13,byte 13,byte 13,byte 12,byte 12,byte 12,byte 12, + byte 15,byte 15,byte 15,byte 15,byte 14,byte 14,byte 14,byte 14, + byte 13,byte 13,byte 13,byte 13,byte 12,byte 12,byte 12,byte 12, + byte 11,byte 11,byte 11,byte 11,byte 10,byte 10,byte 10,byte 10, + byte 9,byte 9,byte 9,byte 9,byte 8,byte 8,byte 8,byte 8, + byte 11,byte 11,byte 11,byte 11,byte 10,byte 10,byte 10,byte 10, + byte 9,byte 9,byte 9,byte 9,byte 8,byte 8,byte 8,byte 8, + byte 11,byte 11,byte 11,byte 11,byte 10,byte 10,byte 10,byte 10, + byte 9,byte 9,byte 9,byte 9,byte 8,byte 8,byte 8,byte 8, + byte 11,byte 11,byte 11,byte 11,byte 10,byte 10,byte 10,byte 10, + byte 9,byte 9,byte 9,byte 9,byte 8,byte 8,byte 8,byte 8, + byte 7,byte 7,byte 7,byte 7,byte 6,byte 6,byte 6,byte 6, + byte 5,byte 5,byte 5,byte 5,byte 4,byte 4,byte 4,byte 4, + byte 7,byte 7,byte 7,byte 7,byte 6,byte 6,byte 6,byte 6, + byte 5,byte 5,byte 5,byte 5,byte 4,byte 4,byte 4,byte 4, + byte 7,byte 7,byte 7,byte 7,byte 6,byte 6,byte 6,byte 6, + byte 5,byte 5,byte 5,byte 5,byte 4,byte 4,byte 4,byte 4, + byte 7,byte 7,byte 7,byte 7,byte 6,byte 6,byte 6,byte 6, + byte 5,byte 5,byte 5,byte 5,byte 4,byte 4,byte 4,byte 4, + byte 3,byte 3,byte 3,byte 3,byte 2,byte 2,byte 2,byte 2, + byte 1,byte 1,byte 1,byte 1,byte 0,byte 0,byte 0,byte 0, + byte 3,byte 3,byte 3,byte 3,byte 2,byte 2,byte 2,byte 2, + byte 1,byte 1,byte 1,byte 1,byte 0,byte 0,byte 0,byte 0, + byte 3,byte 3,byte 3,byte 3,byte 2,byte 2,byte 2,byte 2, + byte 1,byte 1,byte 1,byte 1,byte 0,byte 0,byte 0,byte 0, + byte 3,byte 3,byte 3,byte 3,byte 2,byte 2,byte 2,byte 2, + byte 1,byte 1,byte 1,byte 1,byte 0,byte 0,byte 0,byte 0, +}; + +rgbvmap := array[3*256] of { + byte 255,byte 255,byte 255, byte 255,byte 255,byte 170, + byte 255,byte 255,byte 85, byte 255,byte 255,byte 0, + byte 255,byte 170,byte 255, byte 255,byte 170,byte 170, + byte 255,byte 170,byte 85, byte 255,byte 170,byte 0, + byte 255,byte 85,byte 255, byte 255,byte 85,byte 170, + byte 255,byte 85,byte 85, byte 255,byte 85,byte 0, + byte 255,byte 0,byte 255, byte 255,byte 0,byte 170, + byte 255,byte 0,byte 85, byte 255,byte 0,byte 0, + byte 238,byte 0,byte 0, byte 238,byte 238,byte 238, + byte 238,byte 238,byte 158, byte 238,byte 238,byte 79, + byte 238,byte 238,byte 0, byte 238,byte 158,byte 238, + byte 238,byte 158,byte 158, byte 238,byte 158,byte 79, + byte 238,byte 158,byte 0, byte 238,byte 79,byte 238, + byte 238,byte 79,byte 158, byte 238,byte 79,byte 79, + byte 238,byte 79,byte 0, byte 238,byte 0,byte 238, + byte 238,byte 0,byte 158, byte 238,byte 0,byte 79, + byte 221,byte 0,byte 73, byte 221,byte 0,byte 0, + byte 221,byte 221,byte 221, byte 221,byte 221,byte 147, + byte 221,byte 221,byte 73, byte 221,byte 221,byte 0, + byte 221,byte 147,byte 221, byte 221,byte 147,byte 147, + byte 221,byte 147,byte 73, byte 221,byte 147,byte 0, + byte 221,byte 73,byte 221, byte 221,byte 73,byte 147, + byte 221,byte 73,byte 73, byte 221,byte 73,byte 0, + byte 221,byte 0,byte 221, byte 221,byte 0,byte 147, + byte 204,byte 0,byte 136, byte 204,byte 0,byte 68, + byte 204,byte 0,byte 0, byte 204,byte 204,byte 204, + byte 204,byte 204,byte 136, byte 204,byte 204,byte 68, + byte 204,byte 204,byte 0, byte 204,byte 136,byte 204, + byte 204,byte 136,byte 136, byte 204,byte 136,byte 68, + byte 204,byte 136,byte 0, byte 204,byte 68,byte 204, + byte 204,byte 68,byte 136, byte 204,byte 68,byte 68, + byte 204,byte 68,byte 0, byte 204,byte 0,byte 204, + byte 170,byte 255,byte 170, byte 170,byte 255,byte 85, + byte 170,byte 255,byte 0, byte 170,byte 170,byte 255, + byte 187,byte 187,byte 187, byte 187,byte 187,byte 93, + byte 187,byte 187,byte 0, byte 170,byte 85,byte 255, + byte 187,byte 93,byte 187, byte 187,byte 93,byte 93, + byte 187,byte 93,byte 0, byte 170,byte 0,byte 255, + byte 187,byte 0,byte 187, byte 187,byte 0,byte 93, + byte 187,byte 0,byte 0, byte 170,byte 255,byte 255, + byte 158,byte 238,byte 238, byte 158,byte 238,byte 158, + byte 158,byte 238,byte 79, byte 158,byte 238,byte 0, + byte 158,byte 158,byte 238, byte 170,byte 170,byte 170, + byte 170,byte 170,byte 85, byte 170,byte 170,byte 0, + byte 158,byte 79,byte 238, byte 170,byte 85,byte 170, + byte 170,byte 85,byte 85, byte 170,byte 85,byte 0, + byte 158,byte 0,byte 238, byte 170,byte 0,byte 170, + byte 170,byte 0,byte 85, byte 170,byte 0,byte 0, + byte 153,byte 0,byte 0, byte 147,byte 221,byte 221, + byte 147,byte 221,byte 147, byte 147,byte 221,byte 73, + byte 147,byte 221,byte 0, byte 147,byte 147,byte 221, + byte 153,byte 153,byte 153, byte 153,byte 153,byte 76, + byte 153,byte 153,byte 0, byte 147,byte 73,byte 221, + byte 153,byte 76,byte 153, byte 153,byte 76,byte 76, + byte 153,byte 76,byte 0, byte 147,byte 0,byte 221, + byte 153,byte 0,byte 153, byte 153,byte 0,byte 76, + byte 136,byte 0,byte 68, byte 136,byte 0,byte 0, + byte 136,byte 204,byte 204, byte 136,byte 204,byte 136, + byte 136,byte 204,byte 68, byte 136,byte 204,byte 0, + byte 136,byte 136,byte 204, byte 136,byte 136,byte 136, + byte 136,byte 136,byte 68, byte 136,byte 136,byte 0, + byte 136,byte 68,byte 204, byte 136,byte 68,byte 136, + byte 136,byte 68,byte 68, byte 136,byte 68,byte 0, + byte 136,byte 0,byte 204, byte 136,byte 0,byte 136, + byte 85,byte 255,byte 85, byte 85,byte 255,byte 0, + byte 85,byte 170,byte 255, byte 93,byte 187,byte 187, + byte 93,byte 187,byte 93, byte 93,byte 187,byte 0, + byte 85,byte 85,byte 255, byte 93,byte 93,byte 187, + byte 119,byte 119,byte 119, byte 119,byte 119,byte 0, + byte 85,byte 0,byte 255, byte 93,byte 0,byte 187, + byte 119,byte 0,byte 119, byte 119,byte 0,byte 0, + byte 85,byte 255,byte 255, byte 85,byte 255,byte 170, + byte 79,byte 238,byte 158, byte 79,byte 238,byte 79, + byte 79,byte 238,byte 0, byte 79,byte 158,byte 238, + byte 85,byte 170,byte 170, byte 85,byte 170,byte 85, + byte 85,byte 170,byte 0, byte 79,byte 79,byte 238, + byte 85,byte 85,byte 170, byte 102,byte 102,byte 102, + byte 102,byte 102,byte 0, byte 79,byte 0,byte 238, + byte 85,byte 0,byte 170, byte 102,byte 0,byte 102, + byte 102,byte 0,byte 0, byte 79,byte 238,byte 238, + byte 73,byte 221,byte 221, byte 73,byte 221,byte 147, + byte 73,byte 221,byte 73, byte 73,byte 221,byte 0, + byte 73,byte 147,byte 221, byte 76,byte 153,byte 153, + byte 76,byte 153,byte 76, byte 76,byte 153,byte 0, + byte 73,byte 73,byte 221, byte 76,byte 76,byte 153, + byte 85,byte 85,byte 85, byte 85,byte 85,byte 0, + byte 73,byte 0,byte 221, byte 76,byte 0,byte 153, + byte 85,byte 0,byte 85, byte 85,byte 0,byte 0, + byte 68,byte 0,byte 0, byte 68,byte 204,byte 204, + byte 68,byte 204,byte 136, byte 68,byte 204,byte 68, + byte 68,byte 204,byte 0, byte 68,byte 136,byte 204, + byte 68,byte 136,byte 136, byte 68,byte 136,byte 68, + byte 68,byte 136,byte 0, byte 68,byte 68,byte 204, + byte 68,byte 68,byte 136, byte 68,byte 68,byte 68, + byte 68,byte 68,byte 0, byte 68,byte 0,byte 204, + byte 68,byte 0,byte 136, byte 68,byte 0,byte 68, + byte 0,byte 255,byte 0, byte 0,byte 170,byte 255, + byte 0,byte 187,byte 187, byte 0,byte 187,byte 93, + byte 0,byte 187,byte 0, byte 0,byte 85,byte 255, + byte 0,byte 93,byte 187, byte 0,byte 119,byte 119, + byte 0,byte 119,byte 0, byte 0,byte 0,byte 255, + byte 0,byte 0,byte 187, byte 0,byte 0,byte 119, + byte 51,byte 51,byte 51, byte 0,byte 255,byte 255, + byte 0,byte 255,byte 170, byte 0,byte 255,byte 85, + byte 0,byte 238,byte 79, byte 0,byte 238,byte 0, + byte 0,byte 158,byte 238, byte 0,byte 170,byte 170, + byte 0,byte 170,byte 85, byte 0,byte 170,byte 0, + byte 0,byte 79,byte 238, byte 0,byte 85,byte 170, + byte 0,byte 102,byte 102, byte 0,byte 102,byte 0, + byte 0,byte 0,byte 238, byte 0,byte 0,byte 170, + byte 0,byte 0,byte 102, byte 34,byte 34,byte 34, + byte 0,byte 238,byte 238, byte 0,byte 238,byte 158, + byte 0,byte 221,byte 147, byte 0,byte 221,byte 73, + byte 0,byte 221,byte 0, byte 0,byte 147,byte 221, + byte 0,byte 153,byte 153, byte 0,byte 153,byte 76, + byte 0,byte 153,byte 0, byte 0,byte 73,byte 221, + byte 0,byte 76,byte 153, byte 0,byte 85,byte 85, + byte 0,byte 85,byte 0, byte 0,byte 0,byte 221, + byte 0,byte 0,byte 153, byte 0,byte 0,byte 85, + byte 17,byte 17,byte 17, byte 0,byte 221,byte 221, + byte 0,byte 204,byte 204, byte 0,byte 204,byte 136, + byte 0,byte 204,byte 68, byte 0,byte 204,byte 0, + byte 0,byte 136,byte 204, byte 0,byte 136,byte 136, + byte 0,byte 136,byte 68, byte 0,byte 136,byte 0, + byte 0,byte 68,byte 204, byte 0,byte 68,byte 136, + byte 0,byte 68,byte 68, byte 0,byte 68,byte 0, + byte 0,byte 0,byte 204, byte 0,byte 0,byte 136, + byte 0,byte 0,byte 68, byte 0,byte 0,byte 0, +}; + +clamp: array of int; + +# Remap pixels according to standard Inferno colormap, +# then convert to Inferno compressed image encoding. +# If second component of return is non-nil, it is a compressed mask. +image2enc(i: ref Rawimage, errdiff: int): (array of byte, array of byte, string) +{ + if(sys == nil) + sys = load Sys Sys->PATH; + + j: int; + dx := i.r.max.x-i.r.min.x; + dy := i.r.max.y-i.r.min.y; + cmap := i.cmap; + + if(clamp == nil){ + clamp = array[64+256+64] of int; + for(j=0; j<64; j++) + clamp[j] = 0; + for(j=0; j<256; j++) + clamp[64+j] = (j>>4); + for(j=0; j<64; j++) + clamp[64+256+j] = (255>>4); + } + + pic := i.chans[0]; + npic := len pic; + + maski : ref Rawimage = nil; + if(i.transp) { + mpic := array[npic] of byte; + maski = ref Rawimage ( i.r, nil, 0, byte 0, 1, array[1] of {mpic}, 0, 0 ); + for(j = 0; j < npic; j++) + if(pic[j] == i.trindex) + mpic[j] = byte 0; + else + mpic[j] = byte 1; + } + + + case i.chandesc{ + RImagefile->CRGB1 => + if(cmap == nil) + return (nil, nil, "image has no color map"); + if(i.nchans != 1) + return (nil, nil, sys->sprint("can't handle nchans %d", i.nchans)); + for(j=1; j<=8; j++) + if(len cmap == 3*(1<<j)) + break; + if(j > 8) + return (nil, nil, sys->sprint("can't understand colormap size 3*%d", len cmap/3)); + if(len cmap != 3*256){ + # to avoid a range check in inner loop below, make a full-size cmap + cmap1 := array[3*256] of byte; + cmap1[0:] = cmap[0:]; + cmap = cmap1; + errdiff = 0; # why not? + } + if(errdiff == 0){ + map := array[256] of byte; + k := 0; + for(j=0; j<256; j++){ + r := int cmap[k]>>4; + g := int cmap[k+1]>>4; + b := int cmap[k+2]>>4; + k += 3; + map[j] = byte closest[b+16*(g+16*r)]; + } + for(j=0; j<npic; j++) + pic[j] = map[int pic[j]]; + }else{ + # modified floyd steinberg, coefficients (1 0) 3/16, (0, 1) 3/16, (1, 1) 7/16 + ered := array[dx+1] of int; + egrn := array[dx+1] of int; + eblu := array[dx+1] of int; + for(j=0; j<=dx; j++) + ered[j] = 0; + egrn[0:] = ered[0:]; + eblu[0:] = ered[0:]; + p := 0; + for(y:=0; y<dy; y++){ + er := 0; + eg := 0; + eb := 0; + for(x:=0; x<dx; x++){ + in := 3*int pic[p]; + r := int cmap[in+0]+ered[x]; + g := int cmap[in+1]+egrn[x]; + b := int cmap[in+2]+eblu[x]; + r1 := clamp[r+64]; + g1 := clamp[g+64]; + b1 := clamp[b+64]; + col := int closest[b1+16*(g1+16*r1)]; + pic[p++] = byte col; + + col *= 3; + r -= int rgbvmap[col+0]; + t := (3*r)>>4; + ered[x] = t+er; + ered[x+1] += t; + er = r-3*t; + + g -= int rgbvmap[col+1]; + t = (3*g)>>4; + egrn[x] = t+eg; + egrn[x+1] += t; + eg = g-3*t; + + b -= int rgbvmap[col+2]; + t = (3*b)>>4; + eblu[x] = t+eb; + eblu[x+1] += t; + eb = b-3*t; + } + } + } + RImagefile->CRGB => + if(i.nchans != 3) + return (nil, nil, sys->sprint("RGB image has %d channels", i.nchans)); + rpic := i.chans[0]; + gpic := i.chans[1]; + bpic := i.chans[2]; + if(errdiff == 0){ + for(j=0; j<len rpic; j++){ + r := int rpic[j]>>4; + g := int gpic[j]>>4; + b := int bpic[j]>>4; + pic[j] = byte closest[b+16*(g+16*r)]; + } + }else{ + # modified floyd steinberg, coefficients (1 0) 3/16, (0, 1) 3/16, (1, 1) 7/16 + ered := array[dx+1] of int; + egrn := array[dx+1] of int; + eblu := array[dx+1] of int; + for(j=0; j<=dx; j++) + ered[j] = 0; + egrn[0:] = ered[0:]; + eblu[0:] = ered[0:]; + p := 0; + for(y:=0; y<dy; y++){ + er := 0; + eg := 0; + eb := 0; + for(x:=0; x<dx; x++){ + r := int rpic[p]+ered[x]; + g := int gpic[p]+egrn[x]; + b := int bpic[p]+eblu[x]; + r1 := clamp[r+64]; + g1 := clamp[g+64]; + b1 := clamp[b+64]; + col := int closest[b1+16*(g1+16*r1)]; + pic[p++] = byte col; + + col *= 3; + r -= int rgbvmap[col+0]; + t := (3*r)>>4; + ered[x] = t+er; + ered[x+1] += t; + er = r-3*t; + + g -= int rgbvmap[col+1]; + t = (3*g)>>4; + egrn[x] = t+eg; + egrn[x+1] += t; + eg = g-3*t; + + b -= int rgbvmap[col+2]; + t = (3*b)>>4; + eblu[x] = t+eb; + eblu[x+1] += t; + eb = b-3*t; + } + } + } + RImagefile->CY => + if(i.nchans != 1) + return (nil, nil, sys->sprint("Y image has %d chans", i.nchans)); + rpic := i.chans[0]; + if(errdiff == 0){ + for(j=0; j<npic; j++){ + r := int rpic[j]>>4; + pic[j] = byte closest[r+16*(r+16*r)]; + } + }else{ + # modified floyd steinberg, coefficients (1 0) 3/16, (0, 1) 3/16, (1, 1) 7/16 + ered := array[dx+1] of int; + for(j=0; j<=dx; j++) + ered[j] = 0; + p := 0; + for(y:=0; y<dy; y++){ + er := 0; + for(x:=0; x<dx; x++){ + r := int rpic[p]+ered[x]; + r1 := clamp[r+64]; + col := int closest[r1+16*(r1+16*r1)]; + pic[p++] = byte col; + + col *= 3; + r -= int rgbvmap[col+0]; + t := (3*r)>>4; + ered[x] = t+er; + ered[x+1] += t; + er = r-3*t; + } + } + } + } + + (encim, estr) := i2e(i); + encmask : array of byte = nil; + if(i.transp && estr == "") + (encmask, estr) = i2e(maski); + return (encim, encmask, estr); +} + +# Some defs from /usr/inferno/include/image.h +NMATCH: con 3; # shortest match possible +NRUN: con (NMATCH+31); # longest match possible +NMEM: con 1024; # window size +NDUMP: con 128; # maximum length of dump +NCBLOCK: con 6000; # size of compressed blocks +MAXCB: con 200; # maximum number of blocks + +HSHIFT: con 3; +NHASH: con (1<<(HSHIFT*NMATCH)); +HMASK: con (NHASH-1); + +Hlist: adt { + index: int; + next: cyclic ref Hlist; + prev: cyclic ref Hlist; +}; + +Outbuf: adt { + hdr: array of byte; + buf: array of byte; + buflen: int; +}; + +i2e(im: ref Rawimage): (array of byte, string) +{ + i : int; + pic := im.chans[0]; + bpl := im.r.max.x - im.r.min.x; + nl := im.r.max.y - im.r.min.y; + edata := bpl * nl; + linei := 0; + hash := array[NHASH] of ref Hlist; + chain := array[NMEM] of ref Hlist; + dumpbuf := array[NDUMP] of byte; + blocks := array[MAXCB] of ref Outbuf; + hdr := array of byte sys->sprint("compressed\n%11d %11d %11d %11d %11d ", + 3, im.r.min.x, im.r.min.y, im.r.max.x, im.r.max.y); + blocks[0] = ref Outbuf(hdr, nil, 0); + outbnum := 1; + y := im.r.min.y; + for(i = 0; i < NHASH; i++) + hash[i] = ref Hlist(0, nil, nil); + for(i = 0; i < NMEM; i++) + chain[i] = ref Hlist(0, nil, nil); + + # if a line is too narrow, we don't compress at all + nmatch := NMATCH; + if(nmatch >= bpl) + nmatch = 0; + + while(linei < edata) { + curblock := ref Outbuf(nil, array[NCBLOCK] of byte, 0); + blocks[outbnum] = curblock; + outbuf := curblock.buf; + cp := 0; + h := 0; + for(i = 0; i < NHASH; i++) { + hi := hash[i]; + hi.next = nil; + hi.prev = nil; + } + for(i = 0; i < NMEM; i++) { + ci := chain[i]; + ci.next = nil; + ci.prev = nil; + } + + outp := 0; + for(i = 0; i <= nmatch; i++) + h = (((h<<HSHIFT)^(int pic[linei+i]))&HMASK); + loutp := 0; + + blockloop: + while(linei < edata) { + ndump := 0; + eline := linei + bpl; + for(p := linei; p < eline;) { + es : int; + if(eline - p < NRUN) + es = eline; + else + es = p + NRUN; + q := 0; + runlen := 2; + hp := hash[h]; + matchloop: + for(hp=hp.next; hp != nil; hp = hp.next) { + s := p + runlen; + if(s >= es) + continue matchloop; + t := hp.index + runlen; + for(; s >= p; s--){ + if(pic[s] != pic[t]) + continue matchloop; + t--; + } + t += runlen+2; + s += runlen+2; + for(; s < es; s++) { + if(pic[s] != pic[t]) + break; + t++; + } + n := s - p; + if(n > runlen) { + runlen = n; + q = hp.index; + if(n == NRUN) + break; + } + } + if(runlen < NMATCH) { + if(ndump == NDUMP) { + if(NCBLOCK-outp < NDUMP+1) + break blockloop; + outbuf[outp++] = byte (NDUMP-1+128); + outbuf[outp:] = dumpbuf; + outp += NDUMP; + ndump = 0; + } + dumpbuf[ndump++] = pic[p]; + runlen = 1; + } + else { + if(ndump != 0) { + if(NCBLOCK-outp < ndump+1) + break blockloop; + outbuf[outp++] = byte (ndump-1+128); + outbuf[outp:] = dumpbuf[0:ndump]; + outp += ndump; + ndump = 0; + } + offs := p - q - 1; + if(NCBLOCK-outp < 2) + break blockloop; + outbuf[outp++] = byte (((runlen-NMATCH)<<2)+(offs>>8)); + outbuf[outp++] = byte offs; + } + for(q = p+runlen; p < q; p++) { + c := chain[cp]; + if(c.prev != nil) + c.prev.next = nil; + c.next = hash[h].next; + c.prev = hash[h]; + if(c.next != nil) + c.next.prev = c; + c.prev.next = c; + c.index = p; + cp++; + if(cp == NMEM) + cp = 0; + if(edata-p > NMATCH) + h = (((h<<HSHIFT)^(int pic[p+NMATCH]))&HMASK); + } + } + if(ndump != 0) { + if(NCBLOCK-outp < ndump+1) + break blockloop; + outbuf[outp++] = byte (ndump-1+128); + outbuf[outp:] = dumpbuf[0:ndump]; + outp += ndump; + } + linei = eline; + loutp = outp; + y++; + } + + # current block output buffer full + if(loutp == 0) + return (nil, "buffer too small for line"); + curblock.hdr = array of byte sys->sprint("%11d %11d ", y, loutp); + curblock.buflen = loutp; + outbnum++; + if(outbnum >= MAXCB) + return (nil, "too many output blocks"); + } + ntot := 0; + for(i = 0; i < outbnum; i++) { + b := blocks[i]; + ntot += len b.hdr + b.buflen; + } + a := array[ntot] of byte; + ai := 0; + for(i = 0; i < outbnum; i++) { + b := blocks[i]; + a[ai:] = b.hdr; + ai += len b.hdr; + if(i > 0) { + a[ai:] = b.buf[0:b.buflen]; + ai += b.buflen; + } + } + return (a, ""); +} diff --git a/appl/svc/webget/image2enc.m b/appl/svc/webget/image2enc.m new file mode 100644 index 00000000..62be121d --- /dev/null +++ b/appl/svc/webget/image2enc.m @@ -0,0 +1,7 @@ +Image2enc: module +{ + PATH: con "/dis/svc/webget/image2enc.dis"; + + image2enc: fn(i: ref RImagefile->Rawimage, errdiff: int): (array of byte, array of byte, string); +}; + diff --git a/appl/svc/webget/message.b b/appl/svc/webget/message.b new file mode 100644 index 00000000..3bf7778b --- /dev/null +++ b/appl/svc/webget/message.b @@ -0,0 +1,249 @@ +implement Message; + +include "sys.m"; + sys: Sys; + +include "string.m"; + S : String; + +include "bufio.m"; + B : Bufio; + Iobuf: import B; + +include "message.m"; + msg: Message; + +msglog: ref Sys->FD; + +init(bufio: Bufio, smod: String) +{ + sys = load Sys Sys->PATH; + S = smod; + B = bufio; +} + +sptab : con " \t"; +crlf : con "\r\n"; + +Msg.newmsg() : ref Msg +{ + return ref Msg("", nil, nil, nil, 0); +} + +# Read a message header from fd and return a Msg +# the header fields. +# If withprefix is true, read one line first and put it +# in the prefixline field of the Msg (without terminating \r\n) +# Return nil if there is a read error or eof before the +# header is completely read. +Msg.readhdr(io: ref Iobuf, withprefix: int) : (ref Msg, string) +{ + m := Msg.newmsg(); + l : list of Nameval = nil; + needprefix := withprefix; + for(;;) { + line := getline(io); + n := len line; + if(n == 0) { + if(withprefix && m.prefixline != "") + break; + return(nil, "msg read hdr error: no header"); + } + if(line[n-1] != '\n') + return (m, "msg read hdr error: incomplete header"); + if(n >= 2 && line[n-2] == '\r') + line = line[0:n-2]; + else + line = line[0:n-1]; + if(needprefix) { + m.prefixline = line; + needprefix = 0; + } + else { + if(line == "") + break; + if(S->in(line[0], sptab)) { + if(l == nil) + continue; + nv := hd l; + l = Nameval(nv.name, nv.value + " " + S->drop(line, sptab)) :: tl l; + } + else { + (nam, val) := S->splitl(line, ":"); + if(val == nil) + continue; # no colon + l = Nameval(S->tolower(nam), S->drop(val[1:], sptab)) :: l; + } + } + } + nh := len l; + if(nh > 0) { + m.fields = array[nh] of Nameval; + for(i := nh-1; i >= 0; i--) { + m.fields[i] = hd l; + l = tl l; + } + } + return (m, ""); +} + +glbuf := array[300] of byte; + +# Like io.gets('\n'), but assume Latin-1 instead of UTF encoding +getline(io: ref Iobuf): string +{ + imax := len glbuf - 1; + for(i := 0; i < imax; ) { + c := io.getb(); + if(c < 0) + break; + if(c < 128) + glbuf[i++] = byte c; + else + i += sys->char2byte(c, glbuf, i); + if(c == '\n') + break; + if(i == imax) { + imax += 100; + if(imax > 1000) + break; # Header lines aren't supposed to be > 1000 + newglbuf := array[imax] of byte; + newglbuf[0:] = glbuf[0:i]; + glbuf = newglbuf; + } + } + ans := string glbuf[0:i]; + return ans; +} + +Bbufsize: con 8000; + +# Read the body of the message, assuming the header has been processed. +# If content-length has been specified, read exactly that many bytes +# or until eof; else read until done. +# Return "" if all is OK, else return an error string. +Msg.readbody(m: self ref Msg, io: ref Iobuf) : string +{ + (clfnd, cl) := m.fieldval("content-length"); + if(clfnd) { + clen := int cl; + if(clen > 0) { + m.body = array[clen] of byte; + n := B->io.read(m.body, clen); + m.bodylen = n; + if(n != clen) + return "short read"; + } + } + else { + m.body = array[Bbufsize] of byte; + curlen := 0; + for(;;) { + avail := len m.body - curlen; + if(avail <= 0) { + newa := array[len m.body + Bbufsize] of byte; + if(curlen > 0) + newa[0:] = m.body[0:curlen]; + m.body = newa; + avail = Bbufsize; + } + n := B->io.read(m.body[curlen:], avail); + if(n < 0) + return sys->sprint("readbody error %r"); + if(n == 0) + break; + else + curlen += n; + } + m.bodylen = curlen; + } + return ""; +} + +# Look for name (lowercase) in the fields of m +# and (1, field value) if found, or (0,"") if not. +# If multiple fields with the same name exist, +# the value is defined as the comma separated list +# of all such values. +Msg.fieldval(m: self ref Msg, name: string) : (int, string) +{ + n := len m.fields; + ans := ""; + found := 0; + for(i := 0; i < n; i++) { + if(m.fields[i].name == name) { + v := m.fields[i].value; + if(found) + ans = ans + ", " + v; + else + ans = v; + found = 1; + } + } + return (found, ans); +} + +Msg.addhdrs(m: self ref Msg, hdrs: list of Nameval) +{ + nh := len hdrs; + if(nh == 0) + return; + onh := len m.fields; + newa := array[nh + onh] of Nameval; + newa[0:] = m.fields; + i := onh; + while(hdrs != nil) { + newa[i++] = hd hdrs; + hdrs = tl hdrs; + } + m.fields = newa; +} + +Msg.update(m: self ref Msg, name, value: string) +{ + for(i := 0; i < len m.fields; i++) + if(m.fields[i].name == name) { + m.fields[i] = Nameval(name, value); + return; + } + m.addhdrs(Nameval(name, value) :: nil); +} + +Msg.header(m: self ref Msg) : string +{ + s := ""; + for(i := 0; i < len m.fields; i++) { + nv := m.fields[i]; + s += nv.name + ": " + nv.value + "\n"; + } + return s; +} + +Msg.writemsg(m: self ref Msg, io: ref Iobuf) : string +{ + n := 0; + if(m.prefixline != nil) { + n = B->io.puts(m.prefixline); + if(n >= 0) + n = B->io.puts(crlf); + } + for(i := 0; i < len m.fields; i++) { + nv := m.fields[i]; + if(n >= 0) + n = B->io.puts(nv.name); + if(n >= 0) + n = B->io.puts(": "); + if(n >= 0) + n = B->io.puts(nv.value); + if(n >= 0) + n = B->io.puts(crlf); + } + if(n >= 0) + n = B->io.puts(crlf); + if(n >= 0 && m.bodylen > 0) + n = B->io.write(m.body, m.bodylen); + if(n < 0) + return sys->sprint("msg write error: %r"); + B->io.flush(); + return ""; +} diff --git a/appl/svc/webget/message.m b/appl/svc/webget/message.m new file mode 100644 index 00000000..04ccfbb3 --- /dev/null +++ b/appl/svc/webget/message.m @@ -0,0 +1,28 @@ +Message: module +{ + PATH: con "/dis/svc/webget/message.dis"; + + init: fn(bufio: Bufio, smod: String); + + Nameval: adt { + name: string; + value: string; + }; + + Msg: adt { + prefixline: string; + prefixbytes: array of byte; + fields: array of Nameval; + body: array of byte; + bodylen: int; + + readhdr: fn(io: ref Bufio->Iobuf, withprefix: int) : (ref Msg, string); + readbody: fn(m: self ref Msg, io: ref Bufio->Iobuf) : string; + writemsg: fn(m: self ref Msg, io: ref Bufio->Iobuf) : string; + header: fn(m: self ref Msg) : string; + addhdrs: fn(m: self ref Msg, hdrs: list of Nameval); + newmsg: fn() : ref Msg; + fieldval: fn(m: self ref Msg, name: string) : (int, string); + update: fn(m: self ref Msg, name, value: string); + }; +}; diff --git a/appl/svc/webget/mkfile b/appl/svc/webget/mkfile new file mode 100644 index 00000000..dcb51917 --- /dev/null +++ b/appl/svc/webget/mkfile @@ -0,0 +1,40 @@ +<../../../mkconfig + +TARG=\ + date.dis\ + file.dis\ + ftp.dis\ + http.dis\ + image2enc.dis\ + message.dis\ + webget.dis\ + wgutils.dis\ + +MODULES=\ + date.m\ + image2enc.m\ + message.m\ + transport.m\ + wgutils.m\ + +SYSMODULES=\ + bufio.m\ + daytime.m\ + draw.m\ + imagefile.m\ + ssl3.m\ + string.m\ + strinttab.m\ + sys.m\ + url.m\ + webget.m\ + +DISBIN=$ROOT/dis/svc/webget + +<$ROOT/mkfiles/mkdis + +install:V: install-logs + +install-logs:V: + rm -f $ROOT/services/webget/webget.log && cp webget.log $ROOT/services/webget/webget.log + # chmod 644 $ROOT/services/webget/webget.log diff --git a/appl/svc/webget/transport.m b/appl/svc/webget/transport.m new file mode 100644 index 00000000..d932f906 --- /dev/null +++ b/appl/svc/webget/transport.m @@ -0,0 +1,5 @@ +Transport: module +{ + init: fn(w: WebgetUtils); + connect: fn(c: ref Fid, r: ref Req, donec: chan of ref Fid); +}; diff --git a/appl/svc/webget/webget.b b/appl/svc/webget/webget.b new file mode 100644 index 00000000..6821303f --- /dev/null +++ b/appl/svc/webget/webget.b @@ -0,0 +1,464 @@ +implement Webget; + +# Protocol +# +# Client opens /chan/webget and writes one of +# GET 0 reqid url types cachectl authcookie\n +# or +# POST bodylength reqid url types cachectl authcookie\n +# body +# +# The possibilities for cachectl are +# max-stale=seconds +# client is willing to accept a response whose age exceeds +# its freshness lifetime (by at most specified seconds) +# without revalidation +# max-age=seconds +# client is unwilling to accept a response whose age +# (now - generation time) exceeds specified seconds +# without revalidiation +# no-cache +# unconditional reload +# Both max-stale and max-age may be specified (separated by comma), +# but no-cache must appear by itself. +# +# Authcookie is optional. If present, it goes in an Authorization: header. +# +# The appropriate transport mechanism gets the entity and +# responds with one of +# OK bodylength reqid type url\n +# body +# or +# ERROR reqid message\n +# +# (In the ERROR case, the message might be "Unauthorized: challenge\n", +# where challenge is of the form "BASIC realm=xxx (param, param, ...)\n". +# The user can be prompted for a name:password, and the request repeated +# with authcookie containing the base64 encoding of name:password). + +include "sys.m"; + sys: Sys; + FD: import sys; + +include "draw.m"; + +include "string.m"; + S: String; + +include "bufio.m"; + B: Bufio; + +include "message.m"; + M: Message; + Msg: import M; + +include "url.m"; + U: Url; + ParsedUrl: import U; + +include "webget.m"; + +include "wgutils.m"; + W: WebgetUtils; + Fid, Req: import W; + +include "transport.m"; + +fhash := array[128] of ref Fid; + +Transports: adt +{ + scheme: int; + m: Transport; +}; +transports: array of ref Transports; + +transtab := array[] of { + (Url->HTTP, "/dis/svc/webget/http.dis"), + (Url->HTTPS, nil), # nil means: same as previous + (Url->FILE, "/dis/svc/webget/file.dis"), + (Url->FTP, "/dis/svc/webget/ftp.dis") +}; + +Transpreq: adt +{ + index: int; + fid: ref Fid; + r: ref Req; + next: cyclic ref Transpreq; +}; + +Rchunk: con 30; +# Transpmax: con 5; # max number of simultaneously spawned transports +Transpmax: con 1; # max number of simultaneously spawned transports + +logfile: con "/services/webget/webget.log"; +DO_LOG: con 1; + +stderr: ref FD; + +# to start ever-present webget +init(nil: ref Draw->Context, nil: list of string) +{ + dummyctl := chan of int; + spawn start(dummyctl); + <- dummyctl; + <- dummyctl; +} + +# sends a 1 on ctl when ready to serve, +# 0 if there was some problem starting. +start(ctl: chan of int) +{ + sys = load Sys Sys->PATH; + stderr = sys->fildes(2); + ok := 1; + ntransp := 0; + tqueuehd: ref Transpreq = nil; + tqueuetl: ref Transpreq = nil; + + log : ref Sys->FD; + if(DO_LOG) + log = sys->create(logfile, Sys->OWRITE, 8r666); + + io := sys->file2chan("/chan", "webget"); + if(io == nil) { + sys->fprint(stderr, "webget: failed to post: %r\n"); + ok = 0; + } + + B = load Bufio Bufio->PATH; + if(B == nil) { + sys->fprint(stderr, "webget: failed to load Bufio: %r\n"); + ok = 0; + } + S = load String String->PATH; + if(S == nil) { + sys->fprint(stderr, "webget: failed to load String: %r\n"); + ok = 0; + } + M = load Message Message->PATH; + if(M == nil) { + sys->fprint(stderr, "webget: failed to load Message: %r\n"); + ok = 0; + } + M->init(B, S); + U = load Url Url->PATH; + if(U == nil) { + sys->fprint(stderr, "webget: failed to load Url: %r\n"); + ok = 0; + } + U->init(); + W = load WebgetUtils WebgetUtils->PATH; + if(W == nil) { + sys->fprint(stderr, "webget: failed to load WebgetUtils: %r\n"); + ok = 0; + } + if(!ok) { + ctl <-= 0; + return; + } + W->init(M, S, B, U, log); + + loadtransmod(); + + donec := chan of ref Fid; + ctl <-= 1; + + + altloop: + for(;;) alt { + (nil, data, fid, wc) := <-io.write => + if(wc == nil) { + finish(fid); + continue altloop; + } + ndata := len data; + c := lookup(fid); + W->log(c, "\nREQUEST: " + string data); + iw := c.writer; + n := len c.reqs; + if(iw == n) { + newrs := array[n + Rchunk] of ref Req; + newrs[0:] = c.reqs[0:n]; + c.reqs = newrs; + } + r := c.reqs[iw]; + err := ""; + if(r == nil) { + # initial part (or all) of a request + r = ref Req(iw, "", 0, "", "", "", "", "", nil, nil, nil); + c.reqs[iw] = r; + + # expect at least the prefix line to be in data + prefix := ""; + for(i := 0; i < ndata; i++) { + if(int data[i] == '\n') { + prefix = string data[0:i]; + if(i+1 < ndata) { + r.body = array[ndata-i-1] of byte; + r.body[0:] = data[i:ndata]; + } + break; + } + } + if(prefix == "") + err = "no prefix line"; + else if(prefix == "FINISH") { + writereply(wc, len data, ""); + finish(fid); + continue altloop; + } + else { + (nl, l) := sys->tokenize(prefix, "∎"); + if(nl != 6 && nl != 7) + err = "wrong number of fields in " + prefix; + else { + r.method = hd l; + l = tl l; + r.bodylen = int hd(l); + l = tl l; + r.reqid = hd l; + l = tl l; + r.loc = hd l; + l = tl l; + r.types = hd l; + l = tl l; + r.cachectl = hd l; + l = tl l; + if(l != nil) + r.auth = hd l; + locurl := U->makeurl(r.loc); + if(locurl.scheme == U->MAILTO) + err = "webget shouldn't get mailto"; + else if(locurl.scheme == U->NOSCHEME || + (locurl.scheme != U->FILE && (locurl.host == "" || locurl.pstart != "/"))) + err = "url not absolute: " + r.loc; + r.url = locurl; + } + } + if(err != "") + err = "webget protocol error: " + err; + } + else { + # continuation of request: more body + olen := len r.body; + newa := array[olen + ndata] of byte; + newa[0:] = r.body[0:olen]; + newa[olen:] = data[0:ndata]; + r.body = newa; + } + if(err == "" && len r.body == r.bodylen) { + # request complete: spawn off transport handler + c.writer++; + scheme := r.url.scheme; + found := 0; + for(i := 0; i < len transports; i++) { + if(transports[i].scheme == scheme) { + found = 1; + break; + } + } + if(found == 0) + err = "don't know how to fetch " + r.loc; + else { + if(ntransp < Transpmax) { + W->log(c, "transport " + string scheme + ": get " + r.loc); + spawn transports[i].m->connect(c, r, donec); + ntransp++; + } + else { + # too many active transports: queue this until later + tr := ref Transpreq(i, c, r, nil); + if(tqueuetl == nil) + tqueuehd = tqueuetl = tr; + else { + tqueuetl.next = tr; + tqueuetl = tr; + } + } + } + } + if(err != "") { + writereply(wc, -1, err); + W->log(c, err); + c.reqs[iw] = nil; + } + else + writereply(wc, ndata, ""); + + (nil, nbyte, fid, rc) := <-io.read => + if(rc == nil) { + finish(fid); + continue altloop; + } + c := lookup(fid); + c.nbyte = nbyte; + c.rc = rc; + readans(c); + c := <- donec => + ntransp--; + if(tqueuehd != nil) { + tr := tqueuehd; + tqueuehd = tr.next; + if(tqueuehd == nil) + tqueuetl = nil; + W->log(c, "transport: get " + tr.r.loc); + spawn transports[tr.index].m->connect(tr.fid, tr.r, donec); + ntransp++; + } + readans(c); + c = nil; + } +} + +loadtransmod() +{ + transports = array[len transtab] of ref Transports; + j := 0; + prevt : ref Transports = nil; + for(i := 0; i < len transtab; i++) { + (scheme, path) := transtab[i]; + if(path == nil) { + if(prevt != nil) + transports[j++] = ref Transports(scheme, prevt.m); + } + else { + t := load Transport path; + if(t == nil) { + sys->fprint(stderr, "failed to load %s: %r\n", path); + continue; + } + + t->init(W); + + ts := ref Transports(scheme, t); + transports[j++] = ts; + prevt = ts; + } + } +} + +# Answer a read request c.nbyte bytes, reply to go to c.rc. +# If c.readr is not -1, it is the index of a req with the currently +# being consumed reply. +# c.nread contains the number of bytes of this message read so far. +readans(c: ref Fid) +{ + n := c.nbyte; + if(n <= 0) + return; + ir := c.readr; + if(ir == -1) { + # look for ready reply + for(i := 0; i < c.writer; i++) { + r := c.reqs[i]; + if(r != nil && r.reply != nil) + break; + } + if(i == c.writer) { + return; + } + ir = i; + } + r := c.reqs[ir]; + m := r.reply; + if(m == nil) { + W->log(c, "readans bad state: nil reply"); + readreply(c, nil, ""); + return; + } + if(m.prefixbytes == nil && m.prefixline != "") + m.prefixbytes = array of byte m.prefixline; + plen := len m.prefixbytes; + blen := m.bodylen; + ntot := plen + blen; + nread := c.nread; + if(nread == 0) + W->log(c, "\nREPLY: " + m.prefixline); + nrest := ntot - nread; + if(nrest <= 0) { + W->log(c, "readans bad state: 0 left"); + readreply(c, nil, ""); + return; + } + if(n > nrest) + n = nrest; + n1 := plen - nread; + if(n1 > 0) { + if(n1 > n) + n1 = n; + readreply(c, m.prefixbytes[nread:nread + n1], ""); + c.nread += n1; + } + else { + bpos := nread - plen; + n2 := blen - bpos; + if(n > n2) + n = n2; + readreply(c, m.body[bpos:bpos+n], ""); + c.nread += n; + } + if(c.nread >= ntot) { + c.reqs[ir] = nil; + c.readr = -1; + c.nbyte = 0; + c.nread = 0; + c.rc = nil; + # move back write pointer as far as possible + if(c.writer == ir+1) { + while(ir >= 0 && c.reqs[ir] == nil) + ir--; + c.writer = ir+1; + } + } + else + c.readr = ir; +} + +# Reply to a write command. +writereply(wc: Sys->Rwrite, n: int, err: string) +{ + wc <-= (n, err); +} + +readreply(c: ref Fid, a: array of byte, err: string) +{ + rc := c.rc; + if(rc != nil) + rc <-= (a, err); + c.nbyte = 0; +} + +lookup(fid: int): ref Fid +{ + h := fid%len fhash; + for(f := fhash[h]; f != nil; f = f.link) + if(f.fid == fid) + return f; + f = ref Fid(fid, fhash[h], array[Rchunk] of ref Req, 0, -1, 0, 0, nil); + fhash[h] = f; + + W->log(f, "\nNEW CLIENT"); + + return f; +} + +finish(fid: int) +{ + W->log(nil, "finish"); + h := fid%len fhash; + + prev: ref Fid; + for(f := fhash[h]; f != nil; f = f.link) { + if(f.fid == fid) { + f.rc = nil; + W->log(f, "client finished"); + if(prev == nil) + fhash[h] = f.link; + else + prev.link = f.link; + return; + } + } +} diff --git a/appl/svc/webget/webget.log b/appl/svc/webget/webget.log new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/appl/svc/webget/webget.log diff --git a/appl/svc/webget/wgutils.b b/appl/svc/webget/wgutils.b new file mode 100644 index 00000000..fdf5c375 --- /dev/null +++ b/appl/svc/webget/wgutils.b @@ -0,0 +1,298 @@ +implement WebgetUtils; + +include "sys.m"; + sys: Sys; + +include "draw.m"; + +include "string.m"; + +include "bufio.m"; + +include "imagefile.m"; + readgif, readjpg, readxbitmap: RImagefile; + +include "image2enc.m"; + image2enc: Image2enc; + +include "message.m"; + +include "url.m"; + +include "wgutils.m"; + Iobuf: import B; + +include "strinttab.m"; + T: StringIntTab; + +Msg, Nameval: import M; +ParsedUrl: import U; + +logfd: ref Sys->FD; + +# return from acceptmatch; and conv arg to readbody +BadConv, NoConv, Gif2xcompressed, Jpeg2xcompressed, Xbm2xcompressed: con iota; + +# Both extensions and Content-Types can be in same table. +# This array should be kept sorted +mtypes := array[] of { T->StringInt + ("ai", ApplPostscript), + ("application/html", TextHtml), + ("application/pdf", ApplPdf), + ("application/postscript", ApplPostscript), + ("application/rtf", ApplRtf), + ("application/x-html", TextHtml), + ("au", AudioBasic), + ("audio/au", AudioBasic), + ("audio/basic", AudioBasic), + ("bit", ImageXCompressed), + ("bit2", ImageXCompressed2), + ("eps", ApplPostscript), + ("gif", ImageGif), + ("htm", TextHtml), + ("html", TextHtml), + ("image/gif", ImageGif), + ("image/ief", ImageIef), + ("image/jpeg", ImageJpeg), + ("image/tiff", ImageTiff), + ("image/x-compressed", ImageXCompressed), + ("image/x-compressed2", ImageXCompressed2), + ("image/x-xbitmap", ImageXXBitmap), + ("jpe", ImageJpeg), + ("jpeg", ImageJpeg), + ("jpg", ImageJpeg), + ("pdf", ApplPdf), + ("ps", ApplPostscript), + ("text", TextPlain), + ("text/html", TextHtml), + ("text/plain", TextPlain), + ("text/x-html", TextHtml), + ("tif", ImageTiff), + ("tiff", ImageTiff), + ("txt", TextPlain), + ("video/mpeg", VideoMpeg), + ("video/quicktime", VideoQuicktime), +}; + +# following array must track media type def in wgutils.m +mnames := array[] of { + "application/x-unknown", + "text/plain", + "text/html", + "application/postscript", + "application/rtf", + "application/pdf", + "image/jpeg", + "image/gif", + "image/ief", + "image/tiff", + "image/x-compressed", + "image/x-compressed2", + "image/x-xbitmap", + "audio/basic", + "video/mpeg", + "video/quicktime" +}; + +init(m: Message, s: String, b: Bufio, u: Url, lfd: ref Sys->FD) +{ + sys = load Sys Sys->PATH; + + M = m; + S = s; + B = b; + U = u; + logfd = lfd; + T = load StringIntTab StringIntTab->PATH; + readgif = load RImagefile RImagefile->READGIFPATH; + readjpg = load RImagefile RImagefile->READJPGPATH; + readxbitmap = load RImagefile RImagefile->READXBMPATH; + image2enc = load Image2enc Image2enc->PATH; + if(T == nil || readgif == nil || readjpg == nil || readxbitmap == nil || image2enc == nil) { + sys->fprint(sys->fildes(2), "webget: failed to load T, readgif, readjpg, readxbitmap, or imageremap: %r\n"); + return; + } + readgif->init(B); + readjpg->init(B); + readxbitmap->init(B); +} + +# Return msg with error provoked by bad user action +usererr(r: ref Req, msg: string) : ref Msg +{ + m := Msg.newmsg(); + m.prefixline = sys->sprint("ERROR %s %s", r.reqid, msg); + m.bodylen = 0; + return m; +} + +okprefix(r: ref Req, mrep: ref Msg) +{ + (nil, sctype) := mrep.fieldval("content-type"); + if(sctype == "") + sctype = "text/html"; + else + sctype = canon_mtype(sctype); + (nil, cloc) := mrep.fieldval("content-location"); + if(cloc == "") + cloc = "unknown"; + mrep.prefixline = "OK " + string mrep.bodylen + " " + r.reqid + " " + sctype + " " + cloc; +} + +canon_mtype(s: string) : string +{ + # lowercase, and remove possible parameter + ls := S->tolower(s); + (ts, nil) := S->splitl(ls, "; "); + return ts; +} + +mediatype(s: string, dflt: int) : int +{ + (fnd, val) := T->lookup(mtypes, canon_mtype(s)); + if(!fnd) + val = dflt; + return val; +} + +acceptmatch(ctype: int, accept: string) : int +{ + conv := BadConv; + (nil,l) := sys->tokenize(accept, ","); + while(l != nil) { + a := S->drop(hd l, " \t"); + mt := mediatype(a, -1); + match := (ctype == mt) || (a == "*/*") + || ((ctype == ImageXCompressed || ctype == ImageXCompressed2) + && (mt == ImageJpeg || mt == ImageGif || mt == ImageXXBitmap)); + if(match) { + if(ctype == ImageGif) + conv = Gif2xcompressed; + else if(ctype == ImageJpeg) + conv = Jpeg2xcompressed; + else if(ctype == ImageXXBitmap) + conv = Xbm2xcompressed; + else + conv = NoConv; + break; + } + l = tl l; + } + return conv; +} + +# Get the body of the message whose header is in mrep, +# if io != nil. +# First check that the content type is acceptable. +# Image types will get converted into Inferno compressed format. +# If there is an error, return error string, else "". +# If no error, mrep will contain content-encoding, content-location, +# and content-type fields (guessed if they weren't orignally there). + +getdata(io: ref Iobuf, m: ref Msg, accept: string, url: ref ParsedUrl) : string +{ + (efnd, etype) := m.fieldval("content-encoding"); + if(efnd) + return "content is encoded: " + etype; + ctype := UnknownType; + (tfnd, sctype) := m.fieldval("content-type"); + if(tfnd) + ctype = mediatype(sctype, UnknownType); + else { + # try to guess type from extension + sctype = "(unknown)"; + (nil, name) := S->splitr(url.path, "/"); + if(name != "") { + (f, ext) := S->splitr(name, "."); + if(f != "" && ext != "") { + ctype = mediatype(ext, UnknownType); + if(ctype != UnknownType) { + sctype = mnames[ctype]; + m.update("content-type", sctype); + } + } + } + } + transform := acceptmatch(ctype, accept); + if(transform == BadConv) + return "Unacceptable media type: " + sctype; + (clfnd, cloc) := m.fieldval("content-location"); + if(!clfnd) { + cloc = url.tostring(); + m.update("content-location", cloc); + } + if(transform != NoConv) { + rawimg: ref RImagefile->Rawimage; + err: string; + if(transform == Gif2xcompressed) + (rawimg, err) = readgif->read(io); + else if(transform == Jpeg2xcompressed) + (rawimg, err) = readjpg->read(io); + else if(transform == Xbm2xcompressed) + (rawimg, err) = readxbitmap->read(io); + # if gif file has multiple images, we are supposed to animate, + # but the first one should be there + if(err != "" && err != "ReadGIF: can't handle multiple images in file") + return "error converting image file: " + err; + (data, mask, e) := image2enc->image2enc(rawimg, 1); + if(e != "") + return "error remapping and encoding image file: " + e; + if(mask == nil) + sctype = "image/x-compressed"; + else { + sctype = "image/x-compressed2"; + newdata := array[len data + len mask] of byte; + newdata[0:] = data[0:]; + newdata[len data:] = mask[0:]; + data = newdata; + } + m.body = data; + m.bodylen = len data; + m.update("content-type", sctype); + m.update("content-length", string m.bodylen); + } + else { + # io will be nil if m came from cache + if(io != nil) { + e := m.readbody(io); + if(e != "") + return "reading body: " + e; + } + } + return ""; +} + +# Change an accept spec from webget client into one we can send +# to http server. This means image/x-compressed must be +# changed into image formats we can handle: i.e., gif or jpeg +fixaccept(a: string) : string +{ + (nil,l) := sys->tokenize(a, ","); + ans := ""; + sep := ""; + while(l != nil) { + s := S->drop(hd l, " \t"); + if(s == "image/x-compressed") + ans += sep + "image/gif,image/jpeg,image/x-xbitmap"; + else + ans += sep + s; + sep = ","; + l = tl l; + } + if(ans == "") + ans = "*/*"; + return ans; +} + +log(c: ref Fid, msg: string) +{ + if(logfd != nil) { + # don't use print in case msg is longer than buf + s := ""; + if(c != nil) + s += (string c.fid) + ": "; + s += msg + "\n"; + b := array of byte s; + sys->write(logfd, b, len b); + } +} diff --git a/appl/svc/webget/wgutils.m b/appl/svc/webget/wgutils.m new file mode 100644 index 00000000..5b2458fe --- /dev/null +++ b/appl/svc/webget/wgutils.m @@ -0,0 +1,53 @@ +WebgetUtils: module +{ + PATH: con "/dis/svc/webget/wgutils.dis"; + + Req: adt + { + index: int; + method: string; + bodylen: int; + reqid: string; + loc: string; + types: string; + cachectl: string; + auth: string; + body: array of byte; + url: ref Url->ParsedUrl; + reply: ref Message->Msg; + }; + + Fid: adt + { + fid: int; + link: cyclic ref Fid; + reqs: array of ref Req; + writer: int; + readr: int; + nbyte: int; + nread: int; + rc: Sys->Rread; + }; + + M: Message; + B: Bufio; + S: String; + U: Url; + + # media types (must track mnames array in wgutils.b) + UnknownType, + TextPlain, TextHtml, + ApplPostscript, ApplRtf, ApplPdf, + ImageJpeg, ImageGif, ImageIef, ImageTiff, + ImageXCompressed, ImageXCompressed2, ImageXXBitmap, + AudioBasic, + VideoMpeg, VideoQuicktime: con iota; + + init : fn(m: Message, s: String, b: Bufio, u: Url, logfd: ref Sys->FD); + usererr: fn(r: ref Req, msg: string) : ref Message->Msg; + okprefix: fn(r: ref Req, mrep: ref Message->Msg); + getdata: fn(io: ref Bufio->Iobuf, m: ref Message->Msg, + accept: string, url: ref Url->ParsedUrl) : string; + fixaccept: fn(a: string) : string; + log: fn(c: ref Fid, msg: string); +}; |
