3 * m_getfld.c -- read/parse a message
9 #include <zotnet/mts/mts.h>
11 /* This module has a long and checkered history. First, it didn't burst
12 maildrops correctly because it considered two CTRL-A:s in a row to be
13 an inter-message delimiter. It really is four CTRL-A:s followed by a
14 newline. Unfortunately, MMDF will convert this delimiter *inside* a
15 message to a CTRL-B followed by three CTRL-A:s and a newline. This
16 caused the old version of m_getfld() to declare eom prematurely. The
17 fix was a lot slower than
19 c == '\001' && peekc (iob) == '\001'
21 but it worked, and to increase generality, MBOX style maildrops could
22 be parsed as well. Unfortunately the speed issue finally caught up with
23 us since this routine is at the very heart of MH.
25 To speed things up considerably, the routine Eom() was made an auxilary
26 function called by the macro eom(). Unless we are bursting a maildrop,
27 the eom() macro returns FALSE saying we aren't at the end of the
30 The next thing to do is to read the mts.conf file and initialize
31 delimiter[] and delimlen accordingly...
33 After mhl was made a built-in in msh, m_getfld() worked just fine
34 (using m_unknown() at startup). Until one day: a message which was
35 the result of a bursting was shown. Then, since the burst boundaries
36 aren't CTRL-A:s, m_getfld() would blinding plunge on past the boundary.
37 Very sad. The solution: introduce m_eomsbr(). This hook gets called
38 after the end of each line (since testing for eom involves an fseek()).
39 This worked fine, until one day: a message with no body portion arrived.
42 while (eom (c = Getc (iob), iob))
45 loop caused m_getfld() to return FMTERR. So, that logic was changed to
46 check for (*eom_action) and act accordingly.
48 This worked fine, until one day: someone didn't use four CTRL:A's as
49 their delimiters. So, the bullet got bit and we read mts.h and
50 continue to struggle on. It's not that bad though, since the only time
51 the code gets executed is when inc (or msh) calls it, and both of these
52 have already called mts_init().
54 ------------------------
55 (Written by Van Jacobson for the mh6 m_getfld, January, 1986):
57 This routine was accounting for 60% of the cpu time used by most mh
58 programs. I spent a bit of time tuning and it now accounts for <10%
59 of the time used. Like any heavily tuned routine, it's a bit
60 complex and you want to be sure you understand everything that it's
61 doing before you start hacking on it. Let me try to emphasize
62 that: every line in this atrocity depends on every other line,
63 sometimes in subtle ways. You should understand it all, in detail,
64 before trying to change any part. If you do change it, test the
65 result thoroughly (I use a hand-constructed test file that exercises
66 all the ways a header name, header body, header continuation,
67 header-body separator, body line and body eom can align themselves
68 with respect to a buffer boundary). "Minor" bugs in this routine
69 result in garbaged or lost mail.
71 If you hack on this and slow it down, I, my children and my
72 children's children will curse you.
74 This routine gets used on three different types of files: normal,
75 single msg files, "packed" unix or mmdf mailboxs (when used by inc)
76 and packed, directoried bulletin board files (when used by msh).
77 The biggest impact of different file types is in "eom" testing. The
78 code has been carefully organized to test for eom at appropriate
79 times and at no other times (since the check is quite expensive).
80 I have tried to arrange things so that the eom check need only be
81 done on entry to this routine. Since an eom can only occur after a
82 newline, this is easy to manage for header fields. For the msg
83 body, we try to efficiently search the input buffer to see if
84 contains the eom delimiter. If it does, we take up to the
85 delimiter, otherwise we take everything in the buffer. (The change
86 to the body eom/copy processing produced the most noticeable
87 performance difference, particularly for "inc" and "show".)
89 There are three qualitatively different things this routine busts
90 out of a message: field names, field text and msg bodies. Field
91 names are typically short (~8 char) and the loop that extracts them
92 might terminate on a colon, newline or max width. I considered
93 using a Vax "scanc" to locate the end of the field followed by a
94 "bcopy" but the routine call overhead on a Vax is too large for this
95 to work on short names. If Berkeley ever makes "inline" part of the
96 C optimiser (so things like "scanc" turn into inline instructions) a
97 change here would be worthwhile.
99 Field text is typically 60 - 100 characters so there's (barely)
100 a win in doing a routine call to something that does a "locc"
101 followed by a "bmove". About 30% of the fields have continuations
102 (usually the 822 "received:" lines) and each continuation generates
103 another routine call. "Inline" would be a big win here, as well.
105 Messages, as of this writing, seem to come in two flavors: small
106 (~1K) and long (>2K). Most messages have 400 - 600 bytes of headers
107 so message bodies average at least a few hundred characters.
108 Assuming your system uses reasonably sized stdio buffers (1K or
109 more), this routine should be able to remove the body in large
110 (>500 byte) chunks. The makes the cost of a call to "bcopy"
111 small but there is a premium on checking for the eom in packed
112 maildrops. The eom pattern is always a simple string so we can
113 construct an efficient pattern matcher for it (e.g., a Vax "matchc"
114 instruction). Some thought went into recognizing the start of
115 an eom that has been split across two buffers.
117 This routine wants to deal with large chunks of data so, rather
118 than "getc" into a local buffer, it uses stdio's buffer. If
119 you try to use it on a non-buffered file, you'll get what you
120 deserve. This routine "knows" that struct FILEs have a _ptr
121 and a _cnt to describe the current state of the buffer and
122 it knows that _filbuf ignores the _ptr & _cnt and simply fills
123 the buffer. If stdio on your system doesn't work this way, you
124 may have to make small changes in this routine.
126 This routine also "knows" that an EOF indication on a stream is
127 "sticky" (i.e., you will keep getting EOF until you reposition the
128 stream). If your system doesn't work this way it is broken and you
129 should complain to the vendor. As a consequence of the sticky
130 EOF, this routine will never return any kind of EOF status when
131 there is data in "name" or "buf").
138 static int m_Eom (int, FILE *);
139 static unsigned char *matchc(int, char *, int, char *);
140 static unsigned char *locc(int, unsigned char *, unsigned char);
142 #define Getc(iob) getc(iob)
143 #define eom(c,iob) (msg_style != MS_DEFAULT && \
144 (((c) == *msg_delim && m_Eom(c,iob)) ||\
145 (eom_action && (*eom_action)(c))))
147 static unsigned char **pat_map;
150 * defined in sbr/m_msgdef.c = 0
151 * This is a disgusting hack for "inc" so it can know how many
152 * characters were stuffed in the buffer on the last call
153 * (see comments in uip/scansbr.c).
155 extern int msg_count;
158 * defined in sbr/m_msgdef.c = MS_DEFAULT
160 extern int msg_style;
163 * The "full" delimiter string for a packed maildrop consists
164 * of a newline followed by the actual delimiter. E.g., the
165 * full string for a Unix maildrop would be: "\n\nFrom ".
166 * "Fdelim" points to the start of the full string and is used
167 * in the BODY case of the main routine to search the buffer for
168 * a possible eom. Msg_delim points to the first character of
169 * the actual delim. string (i.e., fdelim+1). Edelim
170 * points to the 2nd character of actual delimiter string. It
171 * is used in m_Eom because the first character of the string
172 * has been read and matched before m_Eom is called.
174 extern char *msg_delim; /* defined in sbr/m_msgdef.c = "" */
175 static unsigned char *fdelim;
176 static unsigned char *delimend;
177 static int fdelimlen;
178 static unsigned char *edelim;
179 static int edelimlen;
181 static int (*eom_action)() = NULL;
184 # define _ptr _p /* Gag */
185 # define _cnt _r /* Retch */
186 # define _filbuf __srget /* Puke */
187 # define DEFINED__FILBUF_TO_SOMETHING_SPECIFIC
193 # define _base __base
194 # define _filbuf(fp) ((fp)->__cnt = 0, __filbuf(fp))
195 # define DEFINED__FILBUF_TO_SOMETHING_SPECIFIC
198 #ifndef DEFINED__FILBUF_TO_SOMETHING_SPECIFIC
199 extern int _filbuf(FILE*);
204 m_getfld (int state, unsigned char *name, unsigned char *buf,
205 int bufsz, FILE *iob)
207 register unsigned char *bp, *cp, *ep, *sp;
208 register int cnt, c, i, j;
210 if ((c = Getc(iob)) < 0) {
217 /* flush null messages */
218 while ((c = Getc(iob)) >= 0 && eom (c, iob))
232 if (c == '\n' || c == '-') {
233 /* we hit the header/body separator */
234 while (c != '\n' && (c = Getc(iob)) >= 0)
237 if (c < 0 || (c = Getc(iob)) < 0 || eom (c, iob)) {
239 /* flush null messages */
240 while ((c = Getc(iob)) >= 0 && eom (c, iob))
253 * get the name of this component. take characters up
254 * to a ':', a newline or NAMESZ-1 characters, whichever
261 bp = sp = (unsigned char *) iob->_IO_read_ptr - 1;
262 j = (cnt = ((long) iob->_IO_read_end -
263 (long) iob->_IO_read_ptr) + 1) < i ? cnt : i;
265 bp = sp = (unsigned char *) iob->_ptr - 1;
266 j = (cnt = iob->_cnt+1) < i ? cnt : i;
268 while (--j >= 0 && (c = *bp++) != ':' && c != '\n')
272 if ((cnt -= j) <= 0) {
274 iob->_IO_read_ptr = iob->_IO_read_end;
275 if (__underflow(iob) == EOF) {
277 if (_filbuf(iob) == EOF) {
280 advise (NULL, "eof encountered in field \"%s\"", name);
284 iob->_IO_read_ptr++; /* NOT automatic in __underflow()! */
288 iob->_IO_read_ptr = bp + 1;
298 * something went wrong. possibilities are:
299 * . hit a newline (error)
300 * . got more than namesz chars. (error)
301 * . hit the end of the buffer. (loop)
305 advise (NULL, "eol encountered in field \"%s\"", name);
311 advise (NULL, "field name \"%s\" exceeds %d bytes", name, NAMESZ - 1);
317 while (isspace (*--cp) && cp >= name)
324 * get (more of) the text of a field. take
325 * characters up to the end of this field (newline
326 * followed by non-blank) or bufsz-1 characters.
328 cp = buf; i = bufsz-1;
331 cnt = (long) iob->_IO_read_end - (long) iob->_IO_read_ptr;
332 bp = (unsigned char *) --iob->_IO_read_ptr;
335 bp = (unsigned char *) --iob->_ptr;
337 c = cnt < i ? cnt : i;
338 while ((ep = locc( c, bp, '\n' ))) {
340 * if we hit the end of this field, return.
342 if ((j = *++ep) != ' ' && j != '\t') {
344 j = ep - (unsigned char *) iob->_IO_read_ptr;
345 memcpy (cp, iob->_IO_read_ptr, j);
346 iob->_IO_read_ptr = ep;
348 j = ep - (unsigned char *) iob->_ptr;
349 memcpy (cp, iob->_ptr, j);
361 * end of input or dest buffer - copy what we've found.
364 c += bp - (unsigned char *) iob->_IO_read_ptr;
365 memcpy( cp, iob->_IO_read_ptr, c);
367 c += bp - (unsigned char *) iob->_ptr;
368 memcpy( cp, iob->_ptr, c);
373 /* the dest buffer is full */
375 iob->_IO_read_ptr += c;
384 * There's one character left in the input buffer.
385 * Copy it & fill the buffer. If the last char
386 * was a newline and the next char is not whitespace,
387 * this is the end of the field. Otherwise loop.
391 *cp++ = j = *(iob->_IO_read_ptr + c);
392 iob->_IO_read_ptr = iob->_IO_read_end;
393 c = __underflow(iob);
394 iob->_IO_read_ptr++; /* NOT automatic! */
396 *cp++ = j = *(iob->_ptr + c);
400 ((j == '\0' || j == '\n') && c != ' ' && c != '\t')) {
418 * get the message body up to bufsz characters or the
419 * end of the message. Sleazy hack: if bufsz is negative
420 * we assume that we were called to copy directly into
421 * the output buffer and we don't add an eos.
423 i = (bufsz < 0) ? -bufsz : bufsz-1;
425 bp = (unsigned char *) --iob->_IO_read_ptr;
426 cnt = (long) iob->_IO_read_end - (long) iob->_IO_read_ptr;
428 bp = (unsigned char *) --iob->_ptr;
431 c = (cnt < i ? cnt : i);
432 if (msg_style != MS_DEFAULT && c > 1) {
434 * packed maildrop - only take up to the (possible)
435 * start of the next message. This "matchc" should
436 * probably be a Boyer-Moore matcher for non-vaxen,
437 * particularly since we have the alignment table
438 * all built for the end-of-buffer test (next).
439 * But our vax timings indicate that the "matchc"
440 * instruction is 50% faster than a carefully coded
441 * B.M. matcher for most strings. (So much for elegant
442 * algorithms vs. brute force.) Since I (currently)
443 * run MH on a vax, we use the matchc instruction. --vj
445 if ((ep = matchc( fdelimlen, fdelim, c, bp )))
449 * There's no delim in the buffer but there may be
450 * a partial one at the end. If so, we want to leave
451 * it so the "eom" check on the next call picks it up.
452 * Use a modified Boyer-Moore matcher to make this
453 * check relatively cheap. The first "if" figures
454 * out what position in the pattern matches the last
455 * character in the buffer. The inner "while" matches
456 * the pattern against the buffer, backwards starting
457 * at that position. Note that unless the buffer
458 * ends with one of the characters in the pattern
459 * (excluding the first and last), we do only one test.
462 if ((sp = pat_map[*ep])) {
465 while (*--ep == *--cp)
470 * ep < bp means that all the buffer
471 * contains is a prefix of delim.
472 * If this prefix is really a delim, the
473 * m_eom call at entry should have found
474 * it. Thus it's not a delim and we can
480 /* try matching one less char of delim string */
482 } while (--sp > fdelim);
486 memcpy( buf, bp, c );
488 iob->_IO_read_ptr += c;
501 adios (NULL, "m_getfld() called with bogus state of %d", state);
505 msg_count = cp - buf;
511 static char unixbuf[BUFSIZ] = "";
521 register char *delimstr;
524 * Figure out what the message delimitter string is for this
525 * maildrop. (This used to be part of m_Eom but I didn't like
526 * the idea of an "if" statement that could only succeed on the
527 * first call to m_Eom getting executed on each call, i.e., at
528 * every newline in the message).
530 * If the first line of the maildrop is a Unix "From " line, we
531 * say the style is MBOX and eat the rest of the line. Otherwise
532 * we say the style is MMDF and look for the delimiter string
533 * specified when nmh was built (or from the mts.conf file).
536 msg_style = MS_UNKNOWN;
539 if (fread (text, sizeof(*text), 5, iob) == 5
540 && strncmp (text, "From ", 5) == 0) {
542 delimstr = "\nFrom ";
544 while ((c = getc (iob)) != '\n' && c >= 0)
548 while ((c = getc (iob)) != '\n' && cp - unixbuf < BUFSIZ - 1)
553 /* not a Unix style maildrop */
554 fseek (iob, pos, SEEK_SET);
555 if (mmdlm2 == NULL || *mmdlm2 == 0)
556 mmdlm2 = "\001\001\001\001\n";
560 c = strlen (delimstr);
561 fdelim = (unsigned char *) malloc((size_t) (c + 3));
564 msg_delim = (char *)fdelim+1;
565 edelim = (unsigned char *)msg_delim+1;
568 strcpy (msg_delim, delimstr);
569 delimend = (unsigned char *)msg_delim + edelimlen;
571 adios (NULL, "maildrop delimiter must be at least 2 bytes");
573 * build a Boyer-Moore end-position map for the matcher in m_getfld.
574 * N.B. - we don't match just the first char (since it's the newline
575 * separator) or the last char (since the matchc would have found it
576 * if it was a real delim).
578 pat_map = (unsigned char **) calloc (256, sizeof(unsigned char *));
580 for (cp = (char *) fdelim + 1; cp < (char *) delimend; cp++ )
581 pat_map[(unsigned char)*cp] = (unsigned char *) cp;
583 if (msg_style == MS_MMDF) {
584 /* flush extra msg hdrs */
585 while ((c = Getc(iob)) >= 0 && eom (c, iob))
594 m_eomsbr (int (*action)())
596 if ((eom_action = action)) {
603 msg_delim = (char *)fdelim + 1;
604 fdelimlen = strlen((char *)fdelim);
605 delimend = (unsigned char *)(msg_delim + edelimlen);
611 * test for msg delimiter string
615 m_Eom (int c, FILE *iob)
617 register long pos = 0L;
625 if ((i = fread (text, sizeof *text, edelimlen, iob)) != edelimlen
626 || strncmp (text, (char *)edelim, edelimlen)) {
627 if (i == 0 && msg_style == MS_MBOX)
628 /* the final newline in the (brain damaged) unix-format
629 * maildrop is part of the delimitter - delete it.
634 fseek (iob, pos, SEEK_SET);
637 fseek (iob, (long)(pos-1), SEEK_SET);
638 getc (iob); /* should be OK */
642 if (msg_style == MS_MBOX) {
644 while ((c = getc (iob)) != '\n')
649 while ((c = getc (iob)) != '\n' && c >= 0 && cp - unixbuf < BUFSIZ - 1)
661 * Return the Return-Path and Delivery-Date
662 * header information.
664 * Currently, I'm assuming that the "From " line
665 * takes one of the following forms.
667 * From sender date remote from host (for UUCP delivery)
668 * From sender@host date (for sendmail delivery)
672 get_returnpath (char *rp, int rplen, char *dd, int ddlen)
674 char *ap, *bp, *cp, *dp;
677 if (!(bp = cp = strchr(ap, ' ')))
681 * Check for "remote from" in envelope to see
682 * if this message uses UUCP style addressing
684 while ((cp = strchr(++cp, 'r'))) {
685 if (strncmp (cp, "remote from", 11) == 0) {
686 cp = strrchr (cp, ' ');
692 * Get the Return-Path information from
693 * the "From " envelope.
696 /* return path for UUCP style addressing */
697 dp = strchr (++cp, '\n');
698 snprintf (rp, rplen, "%.*s!%.*s\n", dp - cp, cp, bp - ap, ap);
700 /* return path for standard domain addressing */
701 snprintf (rp, rplen, "%.*s\n", bp - ap, ap);
705 * advance over the spaces to get to
706 * delivery date on envelope
711 /* Now get delivery date from envelope */
712 snprintf (dd, ddlen, "%.*s\n", 24, bp);
720 static unsigned char *
721 matchc(int patln, char *pat, int strln, char *str)
723 register char *es = str + strln - patln;
726 register char *ep = pat + patln;
727 register char pc = *pat++;
735 while (pp < ep && *sp++ == *pp)
738 return ((unsigned char *)--str);
744 * Locate character "term" in the next "cnt" characters of "src".
745 * If found, return its address, otherwise return 0.
748 static unsigned char *
749 locc(int cnt, unsigned char *src, unsigned char term)
751 while (*src++ != term && --cnt > 0);
753 return (cnt > 0 ? --src : (unsigned char *)0);