WebSVN – SvarDOS – Diff – /svarlang.lib/trunk/tlumacz.c

+}
 /* mvcomp applies the MV-COMPRESSION algorithm to data and returns the compressed size
  * updates len with the number of input bytes left unprocessed */
-static unsigned short mvcomp(void *dstbuf, size_t dstbufsz, const unsigned char *src, size_t *len) {
+static unsigned short mvcomp(void *dstbuf, size_t dstbufsz, const unsigned char *src, size_t *len, unsigned short *maxbytesahead) {
   unsigned short complen = 0;
   unsigned short *dst = dstbuf;
   unsigned short bytesprocessed = 0;
   unsigned char litqueue[32];
   unsigned char litqueuelen = 0;
+  *maxbytesahead = 0;
   /* read src byte by byte, len times, each time look for a match of 15,14,13..2 chars in the back buffer */
   while (*len > 0) {
     unsigned short matchlen;
     unsigned short minmatch;
     unsigned short offset;
     matchlen = 16;
     if (*len < matchlen) matchlen = (unsigned short)(*len);
+    /* monitor the amount of bytes that the compressed stream is "ahead" of
+     * uncompressed data, this is an information used later to size a proper
+     * buffer for in-place depacking */
+    if (complen + litqueuelen + 2 > bytesprocessed) {
+      unsigned short mvstreamlen = complen + litqueuelen + 2;
+      if (*maxbytesahead < (mvstreamlen - bytesprocessed)) *maxbytesahead = mvstreamlen - bytesprocessed;
+    }
     /* abort if no space in output buffer, but do NOT break a literal queue */
     if ((complen >= dstbufsz - 32) && (litqueuelen == 0)) return(complen);
     /* look for a minimum match of 2 bytes, unless I have some pending literal bytes
      * awaiting, in which case I am going through a new data pattern and it is more
+}
 /* write the language block (id, dict, strings) into the LNG file.
  * strings are compressed if compflag != 0 */
-static int svl_write_lang(const struct svl_lang *l, FILE *fd, int compflag) {
+static int svl_write_lang(const struct svl_lang *l, FILE *fd, int compflag, unsigned short *buffrequired) {
   unsigned short strings_bytes = svl_strings_bytes(l);
   unsigned short langid = *((unsigned short *)(&l->id));
   const char *stringsptr = l->strings;
   /* if compressed then do the magic */
   if (compflag) {
     static char compstrings[65000];
     unsigned short comp_bytes;
     size_t stringslen = strings_bytes;
+    unsigned short mvcompbytesahead;
-    comp_bytes = mvcomp(compstrings, sizeof(compstrings), l->strings, &stringslen);
+    comp_bytes = mvcomp(compstrings, sizeof(compstrings), l->strings, &stringslen, &mvcompbytesahead);
+    if (mvcompbytesahead + stringslen > *buffrequired) {
+      *buffrequired = mvcompbytesahead + stringslen;
+    }
     if (comp_bytes < strings_bytes) {
-      printf("lang %c%c mvcomp-ressed (%u bytes -> %u bytes)\n", l->id[0], l->id[1], strings_bytes, comp_bytes);
+      printf("lang %c%c mvcomp-ressed (%u bytes -> %u bytes) mvcomp stream at most %u bytes ahead of raw data (%u bytes needed for in-place decomp)\n", l->id[0], l->id[1], strings_bytes, comp_bytes, mvcompbytesahead, strings_bytes + mvcompbytesahead);
       langid |= 0x8000; /* LNG langblock flag that means "this lang is compressed" */
       strings_bytes = comp_bytes;
       stringsptr = compstrings;
     } else {
       printf("lang %c%c left UNCOMPRESSED (uncomp=%u bytes ; mvcomp=%u bytes)\n", l->id[0], l->id[1], strings_bytes, comp_bytes);
   fd = fopen(fn, "wb");
   if (fd == NULL) {
     return(0);
+  }
-  /* the maximum mvcomp overhead is 3.23% (ie. an increase of 1 byte for every
-   * 31 bytes of source, because 31 source bytes are encoded as a 32 bytes
-   * sequence). The necessary buffer for in-place decompression is therefore
-   * FILESIZE + (FILESIZE / 31) + 1 */
-  allocsz = biggest_langsz + (biggest_langsz / 31) + 1;
+  allocsz = biggest_langsz + (biggest_langsz / 20);
-  printf("biggest lang block is %u bytes -> allocating a %u bytes buffer (3.23%% margin for inplace mvcomp decompression)\n", biggest_langsz, allocsz);
+  printf("biggest lang block is %u bytes -> allocating a %u bytes buffer (5%% safety margin)\n", biggest_langsz, allocsz);
   fprintf(fd, "/* THIS FILE HAS BEEN GENERATED BY TLUMACZ (PART OF THE SVARLANG LIBRARY) */\r\n");
   fprintf(fd, "const unsigned short svarlang_memsz = %uu;\r\n", allocsz);
   fprintf(fd, "const unsigned short svarlang_string_count = %uu;\r\n\r\n", l->num_strings);
   fprintf(fd, "char svarlang_mem[%u] = {\r\n", allocsz);
     /* write lang ID to file, followed string table size, and then
        the dictionary and string table for current language
        skip this for reference language if /excref given */
     if ((reflang != NULL) || (excref == 0)) {
+      /* also updates the biggest_langsz variable to accomodate enough space
+       * for in-place decompression of mvcomp-compressed lang blocks */
-      if (!svl_write_lang(lang, fd, mvcomp_enabled)) {
+      if (!svl_write_lang(lang, fd, mvcomp_enabled, &biggest_langsz)) {
         fprintf(stderr, "ERROR WRITING TO OUTPUT FILE\r\n");
         ecode = 1;
         goto exit_main;
+      }
     } else {

Subversion Repositories SvarDOS

(root)/svarlang.lib/trunk/tlumacz.c – Rev 2117 → 2123