WebSVN – SvarDOS – Diff – /svarlang.lib/trunk/tlumacz.c

+ *
  * usage: tlumacz en fr pl etc
+ *
  * computes an out.lng file that contains all language ressources.
+ *
- * DAT format:
- *
- * 4-bytes signature:
- * "SvL\x1b"
- *
- * Then "LANG BLOCKS" follow. Each LANG BLOCK is prefixed with 4 bytes:
- * II LL    - II is the LANG identifier ("EN", "PL", etc) and LL is the size
- *            of the block (65535 bytes max).
- *
- * Inside a LANG BLOCK is a set of strings:
- *
- * II LL S  where II is the string's 16-bit identifier, LL is its length
- *          (1-65535) and S is the actual string. All strings are ASCIIZ (ie.
- *          they end with a NULL terminator).
- *
- * The list of strings ends with a single 0-long string.
  */
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
+#include <ctype.h>
 #include "svarlang.h"
-struct bitmap {
-  unsigned char bits[8192];
-};
-static void bitmap_set(struct bitmap *b, unsigned short id) {
+#define STRINGS_CAP 65000   /* string storage size in characters */
-  b->bits[id >> 3] |= 1 << (id & 7);
-}
-static int bitmap_get(const struct bitmap *b, unsigned short id) {
+#define DICT_CAP    10000   /* dictionary size in elements */
-  return(b->bits[id >> 3] & (1 << (id & 7)));
-}
-static void bitmap_init(struct bitmap *b) {
-  bzero(b, sizeof(struct bitmap));
-}
 /* read a single line from fd and fills it into dst, returns line length
  * ending CR/LF is trimmed, as well as any trailing spaces */
 static unsigned short readl(char *dst, size_t dstsz, FILE *fd) {
   unsigned short l, lastnonspace = 0;
-  if (fgets(dst, dstsz, fd) == NULL) return(0xffff); /* EOF */
+  if (fgets(dst, (int)dstsz, fd) == NULL) return(0xffff); /* EOF */
   /* trim at first CR or LF and return len */
   for (l = 0; (dst[l] != 0) && (dst[l] != '\r') && (dst[l] != '\n'); l++) {
     if (dst[l] != ' ') lastnonspace = l;
+  }
  * the new length of the string. */
 static unsigned short unesc_string(char *linebuff) {
   unsigned short i;
   for (i = 0; linebuff[i] != 0; i++) {
     if (linebuff[i] != '\\') continue;
-    strcpy(linebuff + i, linebuff + i + 1);
+    memmove(linebuff + i, linebuff + i + 1, strlen(linebuff + i));
     if (linebuff[i] == 0) break;
     switch (linebuff[i]) {
       case 'e':
         linebuff[i] = 0x1B; /* ESC code, using hex because '\e' is not ANSI C */
         break;
+    }
+  }
   return(i);
+}
+#pragma pack(1)
+typedef struct dict_entry {
+    unsigned short id;
+    unsigned short offset;
+} dict_entry_t;
+#pragma pack()
+typedef struct svl_lang {
+  char id[2];
+  unsigned short num_strings;
+  dict_entry_t *dict;
+  size_t dict_cap;
+  char *strings;
+  char *strings_end;
+  size_t strings_cap;
+} svl_lang_t;
+static svl_lang_t * svl_lang_new(char langid[2], size_t dict_cap, size_t strings_cap)
+{
+  svl_lang_t *l;
+  l = malloc(sizeof(svl_lang_t));
+  if (!l) return NULL;
+  l->id[0] = (char)toupper(langid[0]);
+  l->id[1] = (char)toupper(langid[1]);
+  l->dict = malloc(dict_cap * sizeof(dict_entry_t));
+  if (!l->dict) {
+    return NULL;
+  }
+  l->dict_cap = dict_cap;
+  l->num_strings = 0;
+  l->strings = l->strings_end = malloc(strings_cap);
+  if (!l->strings) {
+    free(l->dict);
+    return NULL;
+  }
+  l->strings_cap = strings_cap;
+  return l;
+}
+/* compacts the dict and string buffer */
+static void svl_compact_lang(svl_lang_t *l)
+{
+  size_t bytes;
+  bytes = l->strings_end - l->strings;
+  if (bytes < l->strings_cap) {
+    l->strings = l->strings_end = realloc(l->strings, bytes);
+    l->strings_end += bytes;
+    l->strings_cap = bytes;
+  }
+  l->dict_cap = l->num_strings;
+  l->dict = realloc(l->dict, l->dict_cap * sizeof(dict_entry_t));
+}
+static void svl_lang_free(svl_lang_t *l)
+{
+  l->num_strings = 0;
+  if (l->dict) {
+    free(l->dict);
+    l->dict = NULL;
+  }
+  if (l->strings) {
+    free(l->strings);
+    l->strings = l->strings_end = NULL;
+  }
+  l->dict_cap = 0;
+  l->strings_cap = 0;
+}
+static size_t svl_strings_bytes(svl_lang_t *l)
+{
+  return l->strings_end - l->strings;
+}
+static size_t svl_dict_bytes(svl_lang_t *l)
+{
+  return l->num_strings * sizeof(dict_entry_t);
+}
+static int svl_add_str(svl_lang_t *l, unsigned short id, const char *s)
+{
+  size_t len = strlen(s) + 1;
+  size_t cursor;
+  if (l->strings_cap < svl_strings_bytes(l) + len ||
+      l->dict_cap < (l->num_strings + 1) * sizeof(dict_entry_t)) {
+    return 0;
+  }
+  /* find dictionary insert position, search backwards in assumption
+     that in translation files, strings are generally ordered ascending */
+  for (cursor = l->num_strings; cursor > 0 && l->dict[cursor-1].id > id; cursor--);
+  memmove(&(l->dict[cursor+1]), &(l->dict[cursor]), sizeof(dict_entry_t)*(l->num_strings - cursor));
+  l->dict[cursor].id = id;
+  l->dict[cursor].offset = l->strings_end - l->strings;
+  memcpy(l->strings_end, s, len);
+  l->strings_end += len;
+  l->num_strings++;
+  return 1;
+}
+static int svl_find(svl_lang_t *l, unsigned short id)
+{
+   size_t left = 0, right = l->num_strings - 1, x;
+   unsigned short v;
+   if (l->num_strings == 0) return 0;
+   while (left <= right ) {
+      x = left + ( (right - left ) >> 2 );
+      v = l->dict[x].id;
+      if ( id == v ) return 1;
+      else if ( id > v ) left = x + 1;
+      else right = x - 1;
+   }
+   return 0;
+}
 /* opens a CATS-style file and compiles it into a ressources lang block
  * returns 0 on error, or the size of the generated data block otherwise */
-static unsigned short gen_langstrings(unsigned char *buff, const char *langid, struct bitmap *b, const struct bitmap *refb, const unsigned char *refblock) {
+static unsigned short svl_lang_from_cats_file(svl_lang_t *l, svl_lang_t *refl) {
-  unsigned short len = 0, linelen;
+  unsigned short linelen;
   FILE *fd;
-  char fname[] = "XX.TXT";
+  char fname[] = "xx.txt";
   static char linebuf[8192];
   const char *ptr;
-  unsigned short id, linecount;
+  unsigned short id, maxid=0, maxid_line, linecount;
-  bitmap_init(b);
+  int i;
-  memcpy(fname + strlen(fname) - 6, langid, 2);
+  fname[strlen(fname) - 6] = (char)tolower( l->id[0] );
+  fname[strlen(fname) - 5] = (char)tolower( l->id[1] );
   fd = fopen(fname, "rb");
   if (fd == NULL) {
     printf("ERROR: FAILED TO OPEN '%s'\r\n", fname);
     return(0);
+  }
   for (linecount = 1;; linecount++) {
     linelen = readl(linebuf, sizeof(linebuf), fd);
     if (linelen == 0xffff) break; /* EOF */
     if ((linelen == 0) || (linebuf[0] == '#')) continue;
     /* convert escaped chars to actual bytes (\n -> newline, etc) */
     /* warn about dirty lines */
     if (linebuf[0] == '?') {
       printf("WARNING: %s[#%u] string id %u.%u is flagged as 'dirty'\r\n", fname, linecount, id >> 8, id & 0xff);
+    }
-    /* write string into block (II LL S) */
+    /* add the string contained in current line, if conditions are met */
-    memcpy(buff + len, &id, 2);
+    if (!svl_find(l, id)) {
+      if (refl == NULL || svl_find(refl, id)) {
+        if (!svl_add_str(l, id, ptr)) {
+          printf("ERROR: %s[#%u] output size limit exceeded\r\n", fname, linecount);
+          fclose(fd);
-    len += 2;
+          return 0;
-    {
+        }
-      unsigned short slen = strlen(ptr) + 1;
+        if (id >= maxid) {
+          maxid = id;
-      memcpy(buff + len, &slen, 2);
+          maxid_line = linecount;
-      len += 2;
+        }
-      memcpy(buff + len, ptr, slen);
+        else {
+          printf("WARNING:%s[#%u] file unsorted - line %u has higher id %u.%u\r\n", fname, linecount, maxid_line, maxid >> 8, maxid & 0xff);
-      len += slen;
+        }
-    }
+      }
-    /* if reference bitmap provided: check that the id is valid */
-    if ((refb != NULL) && (bitmap_get(refb, id) == 0)) {
+      else {
-      printf("WARNING: %s[#%u] has an invalid id (%u.%u not present in ref lang)\r\n", fname, linecount, id >> 8, id & 0xff);
+        printf("WARNING: %s[#%u] has an invalid id (%u.%u not present in ref lang)\r\n", fname, linecount, id >> 8, id & 0xff);
-    }
+      }
+    }
-    /* make sure this id is not already present */
-    if (bitmap_get(b, id) == 0) {
-      /* set bit in bitmap to remember I have this string */
-      bitmap_set(b, id);
-    } else {
+    else {
       printf("WARNING: %s[#%u] has a duplicated id (%u.%u)\r\n", fname, linecount, id >> 8, id & 0xff);
+    }
+  }
   fclose(fd);
-  /* if refblock provided, pull missing strings from it */
+  /* if reflang provided, pull missing strings from it */
-  if (refblock != NULL) {
+  if (refl != NULL) {
-    for (;;) {
-      unsigned short slen;
-      id = ((unsigned short *)refblock)[0];
+    for (i = 0; i < refl->num_strings; i++) {
-      slen = ((unsigned short *)refblock)[1];
-      if ((id == 0) && (slen == 0)) break;
+      id = refl->dict[i].id;
-      if (bitmap_get(b, id) == 0) {
+      if (!svl_find(l, id)) {
         printf("WARNING: %s is missing string %u.%u (pulled from ref lang)\r\n", fname, id >> 8, id & 0xff);
-        /* copy missing string from refblock */
+        if (!svl_add_str(l, id, ptr)) {
-        memcpy(buff + len, refblock, slen + 4);
+          printf("ERROR: %s[#%u] output size limit exceeded\r\n", fname, linecount);
-        len += slen + 4;
+          return 0;
+        }
+      }
-      refblock += slen + 4;
+    }
+  }
-  /* write the block terminator (0-long string) */
-  buff[len++] = 0; /* id */
-  buff[len++] = 0; /* id */
+  return(svl_strings_bytes(l));
-  buff[len++] = 0; /* len */
-  buff[len++] = 0; /* len */
-  buff[len++] = 0; /* empty string */
+}
+static int svl_write_header(unsigned short num_strings, FILE *fd)
+{
-  return(len);
+  return (fwrite("SvL1\x1a", 1, 5, fd) == 5) &&
+          (fwrite(&num_strings, 1, 2, fd) == 2);
+}
+static int svl_write_lang(svl_lang_t *l, FILE *fd)
+{
+  unsigned short strings_bytes = svl_strings_bytes(l);
+  return (fwrite(&l->id, 1, 2, fd) == 2) &&
+         (fwrite(&strings_bytes, 1, 2, fd) == 2) &&
+         (fwrite(l->dict, 1, svl_dict_bytes(l), fd) == svl_dict_bytes(l)) &&
+         (fwrite(l->strings, 1, svl_strings_bytes(l), fd) == svl_strings_bytes(l));
+}
+static int svl_write_c_source(svl_lang_t *l, const char *fn, unsigned short biggest_langsz)
+{
+  FILE *fd;
+  int i;
+  unsigned short strings_bytes = svl_strings_bytes(l);
+  unsigned short nextnlat = 0;
+  fd = fopen(fn, "wb");
+  if (fd == NULL) {
+    puts("ERROR: FAILED TO OPEN OR CREATE DEFLANG.C");
+    return 0;
+  } else {
+    unsigned short allocsz = biggest_langsz + (biggest_langsz / 20);
+    printf("biggest lang block is %u bytes -> allocating a %u bytes buffer (5%% safety margin)\n", biggest_langsz,
+           allocsz);
+    fprintf(fd, "/* THIS FILE HAS BEEN GENERATED BY TLUMACZ (PART OF THE SVARLANG LIBRARY) */\r\n");
+    fprintf(fd, "const unsigned short svarlang_memsz = %uu;\r\n", allocsz);
+    fprintf(fd, "const unsigned short svarlang_string_count = %uu;\r\n\r\n", l->num_strings);
+    fprintf(fd, "char svarlang_mem[%u] = {\r\n", allocsz);
+    for (i = 0; i < strings_bytes; i++) {
+      if (!fprintf(fd, "0x%02x", l->strings[i])) {
+        fclose(fd);
+        return 0;
+      }
+      if (i + 1 < strings_bytes) fprintf(fd, ",");
+      nextnlat++;
+      if (l->strings[i] == '\0' || nextnlat == 16) {
+        fprintf(fd, "\r\n");
-#define MEMBLOCKSZ 65000
+        nextnlat = 0;
+      }
+    }
+    fprintf(fd, "};\r\n\r\n");
+    fprintf(fd, "unsigned short svarlang_dict[%u] = {\r\n", l->num_strings * 2);
+    for (i = 0; i < l->num_strings; i++) {
+      if (!fprintf(fd, "0x%04x,0x%04x", l->dict[i].id, l->dict[i].offset)) {
+        fclose(fd);
+        return 0;
+      }
+      if (i + 1 < l->num_strings) fprintf(fd, ",");
+      fprintf(fd, "\r\n");
+    }
+    fprintf(fd, "};\r\n");
+    fclose(fd);
+  }
+  return 1;
+}
 int main(int argc, char **argv) {
   FILE *fd;
   int ecode = 0;
-  char *buff, *refblock;
-  unsigned short refblocksz = 0;
+  svl_lang_t *lang, *reflang = NULL;
-  static struct bitmap bufbitmap;
-  static struct bitmap refbitmap;
-  unsigned short i;
+  int i;
   unsigned short biggest_langsz = 0;
   if (argc < 2) {
     puts("tlumacz ver " SVARLANGVER " - this tool is part of the SvarLANG project.");
     puts("converts a set of CATS-style translations in files EN.TXT, PL.TXT, etc");
     puts("");
     puts("usage: tlumacz en fr pl ...");
     return(1);
+  }
-  buff = malloc(MEMBLOCKSZ);
-  refblock = malloc(MEMBLOCKSZ);
-  if ((buff == NULL) || (refblock == NULL)) {
-    puts("out of memory");
-    return(1);
-  }
   fd = fopen("out.lng", "wb");
   if (fd == NULL) {
     puts("ERR: failed to open or create OUT.LNG");
     return(1);
+  }
-  /* write sig */
-  fwrite("SvL\x1b", 1, 4, fd);
   /* write lang blocks */
   for (i = 1; i < argc; i++) {
     unsigned short sz;
     char id[3];
     if (strlen(argv[i]) != 2) {
       printf("INVALID LANG SPECIFIED: %s\r\n", argv[i]);
       ecode = 1;
       break;
+    }
     id[0] = argv[i][0];
     id[1] = argv[i][1];
     id[2] = 0;
-    if (id[0] >= 'a') id[0] -= 'a' - 'A';
-    if (id[1] >= 'a') id[1] -= 'a' - 'A';
-    sz = gen_langstrings(buff, id, &bufbitmap, (i != 1)?&refbitmap:NULL, (i != 1)?refblock:NULL);
+    if ((lang = svl_lang_new(id, DICT_CAP, STRINGS_CAP)) == NULL) {
+      printf("OUT OF MEMORY\r\n");
+      return(1);
+    }
+    sz = svl_lang_from_cats_file(lang, reflang);
     if (sz == 0) {
       printf("ERROR COMPUTING LANG '%s'\r\n", id);
       ecode = 1;
       break;
     } else {
       printf("computed %s lang block of %u bytes\r\n", id, sz);
       if (sz > biggest_langsz) biggest_langsz = sz;
+    }
+    svl_compact_lang(lang);
-    /* write lang ID to file, followed by block size and then the actual block */
+    /* write header if first (reference) language */
+    if (i == 1) {
-    if ((fwrite(id, 1, 2, fd) != 2) ||
+      if (!svl_write_header(lang->num_strings, fd)) {
-        (fwrite(&sz, 1, 2, fd) != 2) ||
+        printf("ERROR WRITING TO OUTPUT FILE\r\n");
+        ecode = 1;
+        break;
+      }
+    }
+    /* write lang ID to file, followed string table size, and then
+       the dictionary and string table for current language */
-        (fwrite(buff, 1, sz, fd) != sz)) {
+    if (!svl_write_lang(lang, fd)) {
       printf("ERROR WRITING TO OUTPUT FILE\r\n");
       ecode = 1;
       break;
+    }
     /* remember reference data for other languages */
     if (i == 1) {
-      refblocksz = sz;
+      reflang = lang;
+    }
+    else {
-      memcpy(refblock, buff, MEMBLOCKSZ);
+      svl_lang_free(lang);
-      memcpy(&refbitmap, &bufbitmap, sizeof(struct bitmap));
+      lang = NULL;
+    }
+  }
-  fclose(fd);
   /* compute the deflang.c file containing a dump of the reference block */
-  fd = fopen("DEFLANG.C", "wb");
+  if (!svl_write_c_source(reflang, "deflang.c", biggest_langsz)) {
-  if (fd == NULL) {
     puts("ERROR: FAILED TO OPEN OR CREATE DEFLANG.C");
     ecode = 1;
-  } else {
-    unsigned short allocsz = biggest_langsz + (biggest_langsz / 20);
-    unsigned short nextstringin = 0, nextnlat = 40;
-    printf("biggest lang block is %u bytes -> allocating a %u bytes buffer (5%% safety margin)\n", biggest_langsz, allocsz);
-    fprintf(fd, "/* THIS FILE HAS BEEN GENERATED BY TLUMACZ (PART OF THE SVARLANG LIBRARY) */\r\n");
-    fprintf(fd, "const unsigned short svarlang_memsz = %uu;\r\n", allocsz);
-    fprintf(fd, "char svarlang_mem[%u] = {", allocsz);
-    for (i = 0; i < refblocksz; i++) {
-      if (nextstringin == 0) {
-        fprintf(fd, "\r\n");
-        nextnlat = i + 40;
-        nextstringin = 4 + (refblock[i + 3] << 8) + refblock[i + 2];
-        if (nextstringin == 4) nextstringin = 20000; /* last string in block */
-      }
+  }
-      if (i == nextnlat) {
-        nextnlat += 40;
-        fprintf(fd, "\r\n");
-      }
-      nextnlat--;
+  /* clean up */
-      nextstringin--;
+  if (reflang) {
-      fprintf(fd, "%u", refblock[i]);
-      if (i + 1 < refblocksz) fprintf(fd, ",");
-    }
-    fprintf(fd, "};\r\n");
+    svl_lang_free(reflang);
-    fclose(fd);
+    reflang = NULL;
+  }
   return(ecode);
+}

Subversion Repositories SvarDOS

(root)/svarlang.lib/trunk/tlumacz.c – Rev 1272 → 1290