20061220

10 files changed, 686 insertions, 21 deletions

diff --git a/man/2/INDEX b/man/2/INDEX
index 6c7d8e44..f12f0d3a 100644
--- a/man/2/INDEX
+++ b/man/2/INDEX
@@ -21,6 +21,7 @@ btos convcs
 convcs convcs
 stob convcs
 crc crc
+csv csv
 daytime daytime
 filet daytime
 gmt daytime
@@ -98,9 +99,7 @@ inflate filter-deflate
 filter-slip filter-slip
 slip filter-slip
 format format
-
 fsproto fsproto
-fsproto fsproto fsproto
 readprotofile fsproto
 readprotostring fsproto
 geodesy geodesy
@@ -287,6 +286,7 @@ rand rand
 readdir readdir
 regex regex
 registries registries
+rfc822 rfc822
 scsi scsiio
 scsiio scsiio
 secstore secstore
@@ -412,6 +412,7 @@ pctl sys-pctl
 sys-pctl sys-pctl
 pipe sys-pipe
 sys-pipe sys-pipe
+aprint sys-print
 fprint sys-print
 print sys-print
 sprint sys-print
@@ -479,6 +480,8 @@ virgil virgil
 volume volume
 w3c w3c-css
 w3c-css w3c-css
+w3c w3c-uris
+w3c-uris w3c-uris
 w3c w3c-xpointers
 w3c-xpointers w3c-xpointers
 wait wait
diff --git a/man/2/csv b/man/2/csv
new file mode 100644
index 00000000..00b10ffc
--- /dev/null
+++ b/man/2/csv
@@ -0,0 +1,58 @@
+.TH CSV 2
+.SH NAME
+csv \- comma-separated values
+.SH SYNOPSIS
+.EX
+include "bufio.m";
+include "csv.m";
+csv := load CSV CSV->PATH;
+
+init:    fn(bufio: Bufio);
+getline: fn(fd: ref Bufio->Iobuf): list of string;
+quote:   fn(s: string): string;
+.EE
+.SH DESCRIPTION
+.B CSV
+provides a few functions to help read and write data in ``comma-separated value'' format.
+The
+.I csv
+format has a sequence of fields on a line (terminated by a newline, carriage return,
+or carriage return and newline).
+Fields are separated by commas, and may be empty.
+A field that starts with a double-quote is a quoted field, continuing
+to the next lone double-quote character, including commas and line terminators,
+but taking two double-quotes to represent one double-quote.
+Any text following quoted text, up to the next comma or line terminator, is included
+in the field value.
+Note that in a field that does not start with a double-quote, double-quotes are not treated specially.
+.PP
+.B Init
+must be called before any other operation of the module.
+.I Bufio
+is the instance of the
+.IR bufio (2)
+module that will provide the open files to be read.
+.PP
+.B Getline
+reads a sequence of fields in
+.I csv
+format from
+.B Iobuf
+.IR fd ,
+unquotes fields as required, and returns them in order in a list.
+It returns nil on end-of-file (or read error).
+An empty line is regarded as containing a single empty field.
+.PP
+.B Quote
+returns string
+.I s
+quoted as required by the
+.I csv
+format:
+the result is quoted if
+.I s
+contains comma, newline, or a quote (and all embedded quotes are doubled).
+.SH SOURCE
+.B /appl/lib/csv.b
+.SH SEE ALSO
+.IR sh-csv (1)
diff --git a/man/2/fsproto b/man/2/fsproto
index 9f0c5487..d67368ef 100644
--- a/man/2/fsproto
+++ b/man/2/fsproto
@@ -1,6 +1,5 @@
 .TH FSPROTO 2
 .SH NAME
-FSproto: 
 FSproto: readprotofile, readprotostring \- read file system prototype file
 .SH SYNOPSIS
 .EX
diff --git a/man/2/keyring-ipint b/man/2/keyring-ipint
index 8a47c001..d5c3dd8a 100644
--- a/man/2/keyring-ipint
+++ b/man/2/keyring-ipint
@@ -31,6 +31,10 @@ IPint: adt
   cmp:  fn(i1: self ref IPint, i2: ref IPint): int;
   shl:  fn(i: self ref IPint, n: int): ref IPint;
   shr:  fn(i: self ref IPint, n: int): ref IPint;
+  and:  fn(i1: self ref IPint, i2: ref IPint): ref IPint;
+  ior:  fn(i1: self ref IPint, i2: ref IPint): ref IPint;
+  not:  fn(i: self ref IPint): ref IPint;
+  xor:  fn(i1: self ref IPint, i2: ref IPint): ref IPint;
 };
 .EE
 .SH DESCRIPTION
@@ -145,3 +149,28 @@ Returns
 .IB i .shr( n )
 Returns
 .IR i >> n
+.TP
+.IB i1 .and( i2 )
+Returns
+.IR i & n ,
+bitwise AND
+.TP
+.IB i1 .ior( i2 )
+Returns
+.IR i | n ,
+bitwise inclusive-OR
+(it is
+.B ior
+because plain
+.B or
+is a Limbo keyword)
+.TP
+.IB i .not()
+Returns
+.RI ~ i ,
+bitwise ones-complement
+.TP
+.IB i1 .xor( i2 )
+Returns
+.IR i ^ n ,
+bitwise exclusive-OR
diff --git a/man/2/pop3 b/man/2/pop3
index 6288bc2a..938f02c9 100644
--- a/man/2/pop3
+++ b/man/2/pop3
@@ -33,14 +33,9 @@ is
 .IR nil ,
 .B open
 uses
-.BR $POPSERVER ,
-and if that is not defined, it
-uses
-.BR $MAILSERVER ,
-the default mail server, if set up in
-.LR /services/cs/db
-(see
-.IR db (6)).
+.BR $pop3 ,
+which must be defined in
+.IR ndb (6).
 The remaining functions assume a successfully opened connection.
 .PP
 .B Stat
diff --git a/man/2/rfc822 b/man/2/rfc822
new file mode 100644
index 00000000..378d97e5
--- /dev/null
+++ b/man/2/rfc822
@@ -0,0 +1,327 @@
+.TH RFC822 2
+.SH NAME
+rfc822 \- RFC822 mail format helpers
+.SH SYNOPSIS
+.EX
+include "bufio.m";
+include "rfc822.m";
+rfc822 := load RFC822 RFC822->PATH;
+Content, Rfclex: import rfc822;
+
+Word, QString: con ...;
+Maxrequest: con 16*1024;   # more than enough for anything sensible
+
+init:         fn(b: Bufio);
+
+Rfclex: adt {
+   tok:     int;
+   wordval: string;
+
+   mk:      fn(a: array of byte): ref Rfclex;
+   getc:    fn(p: self ref Rfclex): int;
+   ungetc:  fn(p: self ref Rfclex);
+   lex:     fn(p: self ref Rfclex): int;
+   unlex:   fn(p: self ref Rfclex);
+   skipws:  fn(p: self ref Rfclex): int;
+
+   line:    fn(p: self ref Rfclex): string;
+};
+
+readheaders:  fn(fd: ref Bufio->Iobuf, limit: int):
+                  array of (string, array of byte);
+parseparams:  fn(ps: ref Rfclex): list of (string, string);
+parsecontent: fn(ps: ref Rfclex, multipart: int,
+                  head: list of ref Content): list of ref Content;
+mimefields:   fn(ps: ref Rfclex):
+                  list of (string, list of (string, string));
+
+quotable:     fn(s: string): int;
+quote:        fn(s: string): string;
+
+sec2date:     fn(secs: int): string;
+date2sec:     fn(s: string): int;
+now:          fn(): int;
+time:         fn(): string;
+
+Content: adt{
+   generic:   string;
+   specific:  string;
+   params:    list of (string, string);
+
+   mk:        fn(specific: string, generic: string,
+                 params: list of (string, string)): ref Content;
+   check:     fn(c: self ref Content, oks: list of ref Content): int;
+   text:      fn(c: self ref Content): string;
+};
+
+suffixclass:  fn(name: string): (ref Content, ref Content);
+dataclass:    fn(a: array of byte): (ref Content, ref Content);
+.EE
+.SH DESCRIPTION
+.B RFC822
+provides types and functions to help read and parse RFC822 e-mail headers
+(following the more streamlined rules of RFC2822, which has replaced RFC822),
+including some MIME extensions.
+Currently the focus is on operations that support HTTP and other W3C protocols,
+rather than mail reading.
+.PP
+.B Init
+must be called before any other operation in the module.
+It must be given an instance of the
+.B Bufio
+module (see
+.IR bufio (2)).
+.PP
+.B Readheaders
+reads a set of RFC822 header lines from
+.IR fd ,
+ended by an empty line.
+It returns an array of tuples
+.BI ( fieldname,\ value ),
+one per header line.
+The string
+.I fieldname
+is the header line's field name, in lower case.
+The
+.I value
+gives the rest of the line, after removing any initial white space and appending any continuation lines, uninterpreted,
+as an array of bytes (not a string).
+.I Limit
+is the maximum allowed size of the header in bytes;
+usually that is
+.BR Maxrequest .
+.B Readheaders
+returns the tuple
+.B (nil,\ nil)
+on end of file or if the header size limit is exceeded.
+.PP
+.B Rfclex
+takes a header line's
+.I value
+and produces a sequence of tokens.
+It provides the following operations:
+.TP
+.BI Rfclex.mk( a )
+Return a reference to a new
+.B Rfclex
+value that will produce tokens from the array of byte
+.IR a ,
+which is normally the
+.I value
+of one of the header lines returned by
+.BR readheaders .
+.TP
+.IB rfl .getc()
+Return the next character from the input stream, in the Latin-1 (ISO 8859-1) character set.
+Return a negative value
+.RB ( Bufio->EOF )
+on end-of-file.
+.TP
+.IB rfl .ungetc()
+Put back the last character read, which will be returned again by the next call to
+.IR rfl .getc .
+.TP
+.IB rfl .lex()
+Return the next token from the input stream, ignoring any leading white space.
+Most tokens are single characters representing themselves.
+The token value is also assigned to
+.IB rfl .tok .
+There are two special token values:
+.BR Word ,
+representing an unquoted word in the RFC2822 grammar; and
+.BR QString ,
+representing a quoted string.
+In both cases
+.IB rfl .wordval
+will contain the text of the word or string (without quotes).
+.TP
+.IB rfl .unlex()
+Put back the last token read; the next call to
+.IB rfl .lex
+will return it again.
+.TP
+.IB rfl .skipws()
+Skip over any white space at the current position;
+return the initial character of the next token (which is not consumed).
+.TP
+.IB rfl .line()
+Return a string giving the remainder of the input line.
+.PP
+Several functions take an
+.B Rfclex
+referring to a header line's value, parse it, and return a higher-level representation of its value.
+.PP
+.B Parseparams
+parses a sequence of
+.I parameter
+settings of the form (\c
+.BI ; attribute = value\c
+)*
+and returns a corresponding list of tuples
+.BI ( attribute,\ value ) .
+It returns nil if no parameters are found.
+.PP
+.B Parsecontent
+parses the values of fields such as
+.B Content-Type
+and
+.BR Accept .
+The syntax is (loosely) a sequence of comma-separated elements of the form
+.IR type ,
+.IB type /* ,
+or
+.IB type / subtype
+followed by an optional sequence of parameters of the form (\c
+. BI ; attribute = value
+)*.
+The
+.IB type  / subtype
+form is allowed only if
+.I multipart
+is true (non-zero).
+It returns a corresponding list of
+.B Content
+values followed by the initial list
+.IR head .
+.PP
+.B Mimefields
+parses a sequence of comma-separated elements of the form
+.IR word (\c
+.BI ; attr = val
+)*
+as used for instance in the rule
+.IR transfer-coding .
+It returns a corresponding list of tuples
+.BI ( word,\ l )
+where
+.I l
+is an optional list of tuples
+.BI ( attr,\ val ) .
+.PP
+When producing an RFC822 header line, words must be quoted when they contain certain
+special characters.
+.B Quotable
+returns true iff the string
+.I s
+contains any of those characters.
+.B Quote
+returns the value of
+.I s
+with quotes added as required.
+.PP
+RFC822 headers have a particular syntax for dates, different from that of
+.IR daytime (2).
+The function
+.B sec2date
+returns a string in RFC822 date format representing the time
+.I secs
+(in seconds from the Epoch).
+.B Date2sec
+takes a string in RFC822 date format and returns the time in seconds from the Epoch.
+.B Now
+returns the current time in seconds from the epoch
+(it is equivalent to
+.B Daytime->now()
+from
+.IR daytime (2)).
+.B Time
+returns the current time in RFC822's date format.
+.PP
+The Multipurpose Internet Mail Extensions (see RFC2045-7) include syntax
+for describing different types of media, content, and content encodings.
+.B Content
+values represent those descriptions.
+Its fields and operations are as follows:
+.TP
+.B generic
+General class of content (eg,
+.BR application ,
+.BR image ,
+.BR text ,
+etc)
+.TP
+.B specific
+Optional particular type within that class
+(eg,
+.B octet-stream
+within
+.BR application ,
+or
+.B plain
+within
+.BR text )
+.TP
+.B params
+Optional list of
+.BI ( attr,\ value )
+pairs giving parameters to the content type or encoding.
+The particular attribute
+.B q
+has a floating-point
+.I value
+that specifies the relative quality (utility) of a particular type or encoding to a client.
+.TP
+.BI mk( generic,\ specific,\ params )
+Return a reference to a new
+.B Content
+value, initialised with the given parameters.
+.TP
+.IB c .check( ok )
+Return true if
+.I c
+is acceptable content/media/encoding according to the list of allowable forms in
+.IR ok .
+.I C
+is always acceptable if
+.I ok
+is nil (ie, there are no restrictions).
+Otherwise, at least one of the
+.B Content
+values in
+.I ok
+must match
+.IR c .
+That is, an
+.I ok
+value must have the same generic and specific types as
+.IR c ,
+or specify
+.RB ` * '
+in one or both positions (to match any value in
+.IR c ).
+Any
+.I params
+are ignored.
+.TP
+.IB c .text()
+Return the RFC822 syntax for
+.IR c :
+.IB generic / specific ; a\f5=\fIv\fR\&...
+where the component words are quoted if necessary.
+.PP
+Given the
+.I name
+of a file,
+.B suffixclass
+returns a tuple
+.BI ( type,\ enc )
+where
+.I type
+gives the MIME
+.B Content-Type
+of
+.I name
+(or nil, if its type is not known), and
+.I enc
+gives the MIME
+.B Content-Encoding
+of
+.I name
+(or nil, if it is not encoded).
+.SH FILES
+.TF /lib/mimetype
+.TP
+/lib/mimetype
+map between file suffix and MIME content types
+.SH SOURCE
+.B /appl/lib/rfc822.b
diff --git a/man/2/smtp b/man/2/smtp
index 16cde120..f94cdf3d 100644
--- a/man/2/smtp
+++ b/man/2/smtp
@@ -7,7 +7,7 @@ include "smtp.m";
 smtp := load Smtp Smtp->PATH;
 
 open:     fn(server: string): (int, string);
-sendmail: fn(fromwho: string, 
+sendmail: fn(fromwhom: string, 
              towhom: list of string, 
              cc: list of string,
              msg: list of string): (int, string);
@@ -26,11 +26,9 @@ is nil,
 .B open
 uses the
 default mail server
-.B $MAILSERVER
-if set up in
-.BR /services/cs/db ;
-see
-.IR db (6).
+.BR $smtp ,
+which must be defined in
+.IR ndb (6).
 It returns -1 and an error message if the connection fails.
 .PP
 .B Sendmail
diff --git a/man/2/sys-print b/man/2/sys-print
index c499ad43..0fa21403 100644
--- a/man/2/sys-print
+++ b/man/2/sys-print
@@ -1,11 +1,12 @@
 .TH SYS-PRINT 2
 .SH NAME
-print, fprint, sprint \- print formatted output
+print, aprint, fprint, sprint \- print formatted output
 .SH SYNOPSIS
 .EX
 include "sys.m";
 sys := load Sys Sys->PATH;
 
+aprint: fn(format: string, *): array of byte;
 fprint: fn(fd: ref FD, format: string, *): int;
 print:  fn(format: string, *): int;
 sprint: fn(format: string, *): string;
@@ -22,6 +23,10 @@ file descriptor.
 .B Sprint
 places text
 in a string, which it returns.
+.B Aprint
+is similar but returns the text in
+.IR utf (6)
+representation as an array of bytes.
 .B Print
 and
 .B fprint
diff --git a/man/2/w3c-uris b/man/2/w3c-uris
new file mode 100644
index 00000000..d4dbb73d
--- /dev/null
+++ b/man/2/w3c-uris
@@ -0,0 +1,242 @@
+.TH W3C-URIS 2
+.SH NAME
+w3c-uris \- uniform resource identifiers
+.SH SYNOPSIS
+.EX
+include "uris.m";
+
+uris := load URIs URIs->PATH;
+URI: import uris;
+
+URI: adt
+{
+   scheme:     string;
+   userinfo:   string;  # authority, part I
+   host:       string;  # authority, part II
+   port:       string;  # authority, part III
+   path:       string;  # starts with / if absolute
+   query:      string;  # includes ? if not nil
+   fragment:   string;  # includes # if not nil
+
+   parse:      fn(s: string): ref URI;
+   text:       fn(u: self ref URI): string;
+   addbase:    fn(base: self ref URI, rel: ref URI): ref URI;
+   authority:  fn(u: self ref URI): string;
+   copy:       fn(u: self ref URI): ref URI;
+   eq:         fn(u: self ref URI, v: ref URI): int;
+   hasauthority: fn(u: self ref URI): int;
+   isabsolute: fn(u: self ref URI): int;
+   nodots:     fn(u: self ref URI): ref URI;
+   userpw:     fn(u: self ref URI): (string, string);
+};
+
+init: fn();
+dec:  fn(s: string): string;
+enc:  fn(s: string, safe: string): string;
+.EE
+.SH DESCRIPTION
+.B URIs
+supports the `generic syntax' for `Uniform' Resource Identifiers (URIs), defined by RFC3986.
+Each URI can have up to five components in the general syntax:
+.IP
+.IB scheme :
+.BI // authority / path
+.BI ? query
+.BI # fragment
+.PP
+where each component is optional, and can have scheme-specific substructure.
+For instance, in the
+.BR ftp ,
+.B http
+schemes, and perhaps others, the
+.I authority
+component has the further syntax:
+.IP
+.IB userinfo @ host : port
+.PP
+The set of characters allowed in most components is also scheme-specific, as is their interpretation, and indeed the
+interpretation of the component itself.
+.PP
+.B Init
+must be called before any other operation in the module.
+.PP
+.B URI
+represents a parse of a URI into its components, where the
+.I authority
+has been further split into the scheme-specific but common triple of
+.IR userinfo ,
+.I host
+and
+.IR port .
+(The function
+.B URI.authority
+will reproduce the original
+.I authority
+component if required.)
+The
+.B query
+field starts with the
+.RB ` ? '
+character that introduces the
+.I query
+component, so that an empty query is represented by the string
+\f5"?"\fP, and the absence of a query component is represented by a nil value.
+The
+.B fragment
+field is handled in a similar way with its delimiting
+.RB ` # '.
+The fields representing the other components do not include the delimiters in the syntax,
+and all but
+.B query
+have percent-encoded characters decoded.
+(The query string is an exception because the set of characters to escape is application-specific.
+See below for decoding and encoding functions.)
+.B URI
+provides the following operations:
+.TP
+.BI parse( s )
+Return a
+.B URI
+value representing the results of parsing string
+.I s
+as a URI.
+There is no error return.
+The component values have percent-escapes decoded as discussed above.
+The scheme name is converted to lower case.
+.TP
+.IB u .text()
+Return the textual representation of
+.I u
+in the generic syntax,
+adding percent-encoding as required to prevent characters
+being misinterpreted as delimiters.
+.TP
+.IB u .addbase( b )
+Resolves URI reference
+.I u
+with respect to a base URI
+.IR b ,
+including resolving all
+.RB ` . '
+and
+.RB ` .. '
+segments in the URI's path,
+and returns the resulting
+.B URI
+value.
+If
+.I u
+is an absolute URI reference or
+.I b
+is nil, the result is the same as
+.I u
+except that all
+.RB ` . '
+and
+.RB ` .. '
+segments have been resolved in the resulting path, and leading
+instances of them removed.
+.TP
+.IB u .authority()
+Returns the text of the
+.I authority
+component of
+.IR u ,
+in the generic syntax,
+created from its
+.BR userinfo ,
+.B host
+and
+.B port
+components.
+.TP
+.IB u .copy()
+Return a reference to an independent copy of
+.IR u .
+.TP
+.IB u .eq( v )
+Returns true if
+.I u
+and
+.I v
+are textually equal in all components except
+.BR fragment .
+Note that
+.I u
+and
+.I v
+are assumed to be in a canonical  form for the scheme and application.
+.TP
+.IB u .eqf( v )
+Returns true if
+.I u
+and
+.I v
+are textually equal in all components including
+.BR fragment .
+.TP
+.IB u .hasauthority()
+Returns true if any of the authority subcomponents of
+.I u
+are not nil; returns false otherwise.
+.TP
+.IB u .isabsolute()
+Returns true if
+.I u
+has a
+.I scheme
+component; returns false otherwise.
+.TP
+.IB u .nodots()
+Returns a new
+.B URI
+value in which all
+.RB ` . '
+and
+.RB ` .. '
+segments have been resolved (equivalent to
+.IB u .addbase(nil)\c
+).
+.TP
+.IB u .userpw()
+Returns a tuple
+.BI ( username,\ password )
+derived from parsing the
+.I userinfo
+subcomponent of
+.I authority
+using the deprecated but depressingly still common convention that
+.I userinfo
+has the syntax ``\fIusername\fP\f5:\fP\fIpassword\fP''.
+.PP
+A reserved or otherwise special character that appears in a URI component must be
+encoded using a sequence of one or more strings of the form
+.BI % xx
+where
+.I xx
+is the hexadecimal value of one byte of the character.
+A string
+.I s
+containing such encodings can be decoded by the function
+.BR dec .
+A string
+.I s
+can be encoded by
+.BR enc ,
+where the parameter
+.I safe
+lists the characters that need not be escaped (where
+.I safe
+may be nil or empty).
+These functions are normally only needed to decode and encode the values of
+.BR URI.query ,
+because
+.B URI.parse
+and
+.B URI.text
+above decode and encode the other fields.
+.SH SOURCE
+.B /appl/lib/w3c/uris.b
+.SH SEE ALSO
+.IR charon (1),
+.IR httpd (8)
diff --git a/man/2/xml b/man/2/xml
index 8387971a..0b286e49 100644
--- a/man/2/xml
+++ b/man/2/xml
@@ -11,6 +11,8 @@ Parser, Item, Location, Attributes, Mark: import xml;
 init:   fn(): string;
 open: fn(f: string, warning: chan of (Locator, string),
                 preelem: string): (ref Parser, string);
+fopen: fn(iob: ref Bufio->Iobuf, f: string, warning: chan of (Locator, string),
+                preelem: string): (ref Parser, string);
 
 Parser: adt {
     fileoffset: int;
@@ -79,8 +81,15 @@ A new parser instance is created by calling
 .BR open(\fIf\fP,\ \fIwarning\fP,\ \fIpreelem\fP) ,
 which opens the file
 .I f
-for parsing as an XML document.
-It returns a tuple, say
+for parsing as an XML document,
+or
+.BR fopen(\fIiob\fP,\ \fIname\fP,\ \fIwarning\fP,\ \fIpreelem\fP) ,
+which does the same for an already open
+.B Iobuf
+(the string
+.I name
+will be used in diagnostics).
+Both functions return a tuple
 .RI ( p ,\  err ).
 If there is an error opening the document,
 .I p
@@ -94,7 +103,7 @@ If
 is not nil, non-fatal errors encountered when parsing
 will be sent on this channel - a separate process will
 be needed to received them. Each error is represented
-by a tuple, say
+by a tuple
 .RI ( loc ,\  msg ),
 containing the location
 .IR loc ,