Subversion Repositories SvarDOS

Rev

Rev 186 | Go to most recent revision | Only display areas with differences | Ignore whitespace | Details | Blame | Last modification | View Log | RSS feed

Rev 186 Rev 292
1
/*
1
/*
2
  SvarDOS repo index builder
2
  SvarDOS repo index builder
3
  Copyright (C) Mateusz Viste 2012-2021
3
  Copyright (C) Mateusz Viste 2012-2021
4

4

5
  buildidx computes an index tsv file for the SvarDOS repository.
5
  buildidx computes an index tsv file for the SvarDOS repository.
6
  it must be executed pointing to a directory that stores packages (zip)
6
  it must be executed pointing to a directory that stores packages (zip)
7
  files. buildidx will generate the index file and save it into the package
7
  files. buildidx will generate the index file and save it into the package
8
  repository.
8
  repository.
9

9

-
 
10
  11 feb 2021: lsm headers are no longer checked, so it is compatible with the simpler lsm format used by SvarDOS
10
  13 jan 2021: removed the identification line, changed CRC32 to bsum, not creating the listing.txt file and stopped compressing index
11
  13 jan 2021: removed the identification line, changed CRC32 to bsum, not creating the listing.txt file and stopped compressing index
11
  23 apr 2017: uncompressed index is no longer created, added CRC32 of zib (bin only) files, if present
12
  23 apr 2017: uncompressed index is no longer created, added CRC32 of zib (bin only) files, if present
12
  28 aug 2016: listing.txt is always written inside the repo dir (instead of inside current dir)
13
  28 aug 2016: listing.txt is always written inside the repo dir (instead of inside current dir)
13
  27 aug 2016: accepting full paths to repos (starting with /...)
14
  27 aug 2016: accepting full paths to repos (starting with /...)
14
  07 dec 2013: rewritten buildidx in ANSI C89
15
  07 dec 2013: rewritten buildidx in ANSI C89
15
  19 aug 2013: add a compressed version of the index file to repos (index.gz)
16
  19 aug 2013: add a compressed version of the index file to repos (index.gz)
16
  22 jul 2013: creating a listing.txt file with list of packages
17
  22 jul 2013: creating a listing.txt file with list of packages
17
  18 jul 2013: writing the number of packaged into the first line of the lst file
18
  18 jul 2013: writing the number of packaged into the first line of the lst file
18
  11 jul 2013: added a switch to 7za to make it case insensitive when extracting lsm files
19
  11 jul 2013: added a switch to 7za to make it case insensitive when extracting lsm files
19
  10 jul 2013: changed unzip calls to 7za (to handle cases when appinfo is compressed with lzma)
20
  10 jul 2013: changed unzip calls to 7za (to handle cases when appinfo is compressed with lzma)
20
  04 feb 2013: added CRC32 support
21
  04 feb 2013: added CRC32 support
21
  22 sep 2012: forked 1st version from FDUPDATE builder
22
  22 sep 2012: forked 1st version from FDUPDATE builder
22
*/
23
*/
23
 
24
 
24
#include <errno.h>
25
#include <errno.h>
25
#include <stdio.h>   /* fopen, fclose... */
26
#include <stdio.h>   /* fopen, fclose... */
26
#include <stdint.h>
27
#include <stdint.h>
27
#include <stdlib.h>  /* system() */
28
#include <stdlib.h>  /* system() */
28
#include <string.h>  /* strcasecmp() */
29
#include <string.h>  /* strcasecmp() */
29
#include <time.h>    /* time(), ctime() */
30
#include <time.h>    /* time(), ctime() */
30
#include <unistd.h>  /* read() */
31
#include <unistd.h>  /* read() */
31
#include <dirent.h>
32
#include <dirent.h>
32
#include <sys/types.h>
33
#include <sys/types.h>
33
 
34
 
34
#define pVer "2021-01-13"
35
#define pVer "2021-02-11"
35
 
36
 
36
 
37
 
37
/* computes the BSD sum of a file and returns it. returns 0 on error. */
38
/* computes the BSD sum of a file and returns it. returns 0 on error. */
38
static uint16_t file2bsum(const char *filename) {
39
static uint16_t file2bsum(const char *filename) {
39
  uint16_t result = 0;
40
  uint16_t result = 0;
40
  unsigned char buff[1024 * 1024];
41
  unsigned char buff[1024 * 1024];
41
  size_t i, buffread;
42
  size_t i, buffread;
42
  FILE *fd;
43
  FILE *fd;
43
  fd = fopen(filename, "rb");
44
  fd = fopen(filename, "rb");
44
  if (fd == NULL) return(0);
45
  if (fd == NULL) return(0);
45
  while ((buffread = fread(buff, 1, sizeof(buff), fd)) > 0) {
46
  while ((buffread = fread(buff, 1, sizeof(buff), fd)) > 0) {
46
    for (i = 0; i < buffread; i++) {
47
    for (i = 0; i < buffread; i++) {
47
      /* rotr */
48
      /* rotr */
48
      result = (result >> 1) | (result << 15);
49
      result = (result >> 1) | (result << 15);
49
      /* */
50
      /* */
50
      result += buff[i];
51
      result += buff[i];
51
    }
52
    }
52
  }
53
  }
53
  fclose(fd);
54
  fclose(fd);
54
  return(result);
55
  return(result);
55
}
56
}
56
 
57
 
57
 
58
 
58
static void trim(char *str) {
59
static void trim(char *str) {
59
  int x, y, firstchar = -1, lastchar = -1;
60
  int x, y, firstchar = -1, lastchar = -1;
60
  for (x = 0; str[x] != 0; x++) {
61
  for (x = 0; str[x] != 0; x++) {
61
    switch (str[x]) {
62
    switch (str[x]) {
62
      case ' ':
63
      case ' ':
63
      case '\t':
64
      case '\t':
64
      case '\n':
65
      case '\n':
65
      case '\r':
66
      case '\r':
66
        break;
67
        break;
67
      default:
68
      default:
68
        if (firstchar < 0) firstchar = x;
69
        if (firstchar < 0) firstchar = x;
69
        lastchar = x;
70
        lastchar = x;
70
        break;
71
        break;
71
    }
72
    }
72
  }
73
  }
73
  str[lastchar + 1] = 0; /* right trim */
74
  str[lastchar + 1] = 0; /* right trim */
74
  if (firstchar > 0) { /* left trim (shift to the left ) */
75
  if (firstchar > 0) { /* left trim (shift to the left ) */
75
    y = 0;
76
    y = 0;
76
    for (x = firstchar; str[x] != 0; x++) str[y++] = str[x];
77
    for (x = firstchar; str[x] != 0; x++) str[y++] = str[x];
77
    str[y] = 0;
78
    str[y] = 0;
78
  }
79
  }
79
}
80
}
80
 
81
 
81
 
82
 
82
/* reads a line from a file descriptor, and writes it to *line, the *line array is filled no more than maxlen bytes. returns the number of byte read on success, or a negative value on failure (reaching EOF is considered an error condition) */
83
/* reads a line from a file descriptor, and writes it to *line, the *line array is filled no more than maxlen bytes. returns the number of byte read on success, or a negative value on failure (reaching EOF is considered an error condition) */
83
static int readline_fromfile(FILE *fd, char *line, int maxlen) {
84
static int readline_fromfile(FILE *fd, char *line, int maxlen) {
84
  int bytebuff, linelen = 0;
85
  int bytebuff, linelen = 0;
85
  for (;;) {
86
  for (;;) {
86
    bytebuff = fgetc(fd);
87
    bytebuff = fgetc(fd);
87
    if (bytebuff == EOF) {
88
    if (bytebuff == EOF) {
88
      line[linelen] = 0;
89
      line[linelen] = 0;
89
      if (linelen == 0) return(-1);
90
      if (linelen == 0) return(-1);
90
      return(linelen);
91
      return(linelen);
91
    }
92
    }
92
    if (bytebuff < 0) return(-1);
93
    if (bytebuff < 0) return(-1);
93
    if (bytebuff == '\r') continue; /* ignore CR */
94
    if (bytebuff == '\r') continue; /* ignore CR */
94
    if (bytebuff == '\n') {
95
    if (bytebuff == '\n') {
95
      line[linelen] = 0;
96
      line[linelen] = 0;
96
      return(linelen);
97
      return(linelen);
97
    }
98
    }
98
    if (linelen < maxlen) line[linelen++] = bytebuff;
99
    if (linelen < maxlen) line[linelen++] = bytebuff;
99
  }
100
  }
100
}
101
}
101
 
102
 
102
static int readlsm(const char *filename, char *version, char *title, char *description) {
103
static int readlsm(const char *filename, char *version, char *title, char *description) {
103
  char linebuff[1024];
104
  char linebuff[1024];
104
  char *valuestr;
105
  char *valuestr;
105
  int x;
106
  int x;
106
  FILE *fd;
107
  FILE *fd;
107
  /* reset fields to be read to empty values */
108
  /* reset fields to be read to empty values */
108
  version[0] = 0;
109
  version[0] = 0;
109
  description[0] = 0;
110
  description[0] = 0;
110
  title[0] = 0;
111
  title[0] = 0;
111
  /* open the file */
112
  /* open the file */
112
  fd = fopen(filename, "rb");
113
  fd = fopen(filename, "rb");
113
  if (fd == NULL) return(-1);
114
  if (fd == NULL) return(-1);
114
  /* check the file's header */
-
 
115
  if (readline_fromfile(fd, linebuff, 64) < 0) return(-1);
-
 
116
  if (strcasecmp(linebuff, "begin3") != 0) return(-1);
-
 
117
  /* read the LSM file line by line */
115
  /* read the LSM file line by line */
118
  while (readline_fromfile(fd, linebuff, 127) >= 0) {
116
  while (readline_fromfile(fd, linebuff, 127) >= 0) {
119
    for (x = 0;; x++) {
117
    for (x = 0;; x++) {
120
      if (linebuff[x] == 0) {
118
      if (linebuff[x] == 0) {
121
        x = -1;
119
        x = -1;
122
        break;
120
        break;
123
      } else if (linebuff[x] == ':') {
121
      } else if (linebuff[x] == ':') {
124
        break;
122
        break;
125
      }
123
      }
126
    }
124
    }
127
    if (x > 0) {
125
    if (x > 0) {
128
      linebuff[x] = 0;
126
      linebuff[x] = 0;
129
      valuestr = linebuff + x + 1;
127
      valuestr = linebuff + x + 1;
130
      trim(linebuff);
128
      trim(linebuff);
131
      trim(valuestr);
129
      trim(valuestr);
132
      if (strcasecmp(linebuff, "version") == 0) {
130
      if (strcasecmp(linebuff, "version") == 0) {
133
          sprintf(version, "%s", valuestr);
131
          sprintf(version, "%s", valuestr);
134
        } else if (strcasecmp(linebuff, "title") == 0) {
132
        } else if (strcasecmp(linebuff, "title") == 0) {
135
          sprintf(title, "%s", valuestr);
133
          sprintf(title, "%s", valuestr);
136
        } else if (strcasecmp(linebuff, "description") == 0) {
134
        } else if (strcasecmp(linebuff, "description") == 0) {
137
          sprintf(description, "%s", valuestr);
135
          sprintf(description, "%s", valuestr);
138
      }
136
      }
139
    }
137
    }
140
  }
138
  }
141
  fclose(fd);
139
  fclose(fd);
142
  return(0);
140
  return(0);
143
}
141
}
144
 
142
 
145
 
143
 
146
static int cmpstring(const void *p1, const void *p2) {
144
static int cmpstring(const void *p1, const void *p2) {
147
    /* The actual arguments to this function are "pointers to
145
    /* The actual arguments to this function are "pointers to
148
       pointers to char", but strcmp(3) arguments are "pointers
146
       pointers to char", but strcmp(3) arguments are "pointers
149
       to char", hence the following cast plus dereference */
147
       to char", hence the following cast plus dereference */
150
   return(strcasecmp(* (char * const *) p1, * (char * const *) p2));
148
   return(strcasecmp(* (char * const *) p1, * (char * const *) p2));
151
}
149
}
152
 
150
 
153
 
151
 
154
static void GenIndexes(const char *repodir) {
152
static void GenIndexes(const char *repodir) {
155
  char *LsmFileList[4096];
153
  char *LsmFileList[4096];
156
  char tmpbuf[64];
154
  char tmpbuf[64];
157
  char *LsmFile, LSMpackage[64], LSMtitle[128], LSMversion[128], LSMdescription[1024];
155
  char *LsmFile, LSMpackage[64], LSMtitle[128], LSMversion[128], LSMdescription[1024];
158
  int LsmCount = 0, x;
156
  int LsmCount = 0, x;
159
  FILE *idx;
157
  FILE *idx;
160
  DIR *dir;
158
  DIR *dir;
161
  struct dirent *diritem;
159
  struct dirent *diritem;
162
 
160
 
163
  dir = opendir("appinfo");
161
  dir = opendir("appinfo");
164
  if (dir == NULL) {
162
  if (dir == NULL) {
165
    printf("ERROR: Unable to open directory '%s' (%s)\n", repodir, strerror(errno));
163
    printf("ERROR: Unable to open directory '%s' (%s)\n", repodir, strerror(errno));
166
    return;
164
    return;
167
  }
165
  }
168
 
166
 
169
  /* load the content of the appinfo directory */
167
  /* load the content of the appinfo directory */
170
  while ((diritem = readdir(dir)) != NULL) {
168
  while ((diritem = readdir(dir)) != NULL) {
171
    if (strstr(diritem->d_name, ".lsm") == NULL) continue; /* skip files that do not contain '.lsm' */
169
    if (strstr(diritem->d_name, ".lsm") == NULL) continue; /* skip files that do not contain '.lsm' */
172
    LsmFileList[LsmCount] = strdup(diritem->d_name);
170
    LsmFileList[LsmCount] = strdup(diritem->d_name);
173
    /* printf("Loaded LSM file: %s\n", LsmFileList[LsmCount]); */
171
    /* printf("Loaded LSM file: %s\n", LsmFileList[LsmCount]); */
174
    LsmCount += 1;
172
    LsmCount += 1;
175
  }
173
  }
176
 
174
 
177
  closedir(dir);
175
  closedir(dir);
178
 
176
 
179
  printf("Found %d LSM files\n", LsmCount);
177
  printf("Found %d LSM files\n", LsmCount);
180
 
178
 
181
  /* sort the entries */
179
  /* sort the entries */
182
  qsort(&LsmFileList[0], LsmCount, sizeof(char *), cmpstring);
180
  qsort(&LsmFileList[0], LsmCount, sizeof(char *), cmpstring);
183
 
181
 
184
  /* Create the index file */
182
  /* Create the index file */
185
  sprintf(tmpbuf, "%s/index.tsv", repodir);
183
  sprintf(tmpbuf, "%s/index.tsv", repodir);
186
  idx = fopen(tmpbuf, "wb");
184
  idx = fopen(tmpbuf, "wb");
187
 
185
 
188
  /* Read every LSM */
186
  /* Read every LSM */
189
  for (x = 0; x < LsmCount; x++) {
187
  for (x = 0; x < LsmCount; x++) {
190
    uint16_t bsum;
188
    uint16_t bsum;
191
    LsmFile = LsmFileList[x];
189
    LsmFile = LsmFileList[x];
192
    sprintf(LSMpackage, "%s", LsmFile);
190
    sprintf(LSMpackage, "%s", LsmFile);
193
    LSMpackage[strlen(LSMpackage) - 4] = 0;
191
    LSMpackage[strlen(LSMpackage) - 4] = 0;
194
 
192
 
195
    /* compute the BSD sum of the zip package */
193
    /* compute the BSD sum of the zip package */
196
    sprintf(tmpbuf, "%s/%s.zip", repodir, LSMpackage);
194
    sprintf(tmpbuf, "%s/%s.zip", repodir, LSMpackage);
197
    bsum = file2bsum(tmpbuf);
195
    bsum = file2bsum(tmpbuf);
198
 
196
 
199
    printf("Processing %s... BSUM %04X\n", LsmFile, bsum);
197
    printf("Processing %s... BSUM %04X\n", LsmFile, bsum);
200
 
198
 
201
    sprintf(tmpbuf, "appinfo/%s", LsmFile);
199
    sprintf(tmpbuf, "appinfo/%s", LsmFile);
202
    readlsm(tmpbuf, LSMversion, LSMtitle, LSMdescription);
200
    readlsm(tmpbuf, LSMversion, LSMtitle, LSMdescription);
203
 
201
 
204
    if (strlen(LSMpackage) > 8) printf("Warning: %s.zip is not in 8.3 format!\n", LSMpackage);
202
    if (strlen(LSMpackage) > 8) printf("Warning: %s.zip is not in 8.3 format!\n", LSMpackage);
205
    if (LSMtitle[0] == 0) printf("Warning: no LSM title for %s.zip\n", LSMpackage);
203
    if (LSMtitle[0] == 0) printf("Warning: no LSM title for %s.zip\n", LSMpackage);
206
    if (LSMversion[0] == 0) printf("Warning: no LSM version for %s.zip!\n", LSMpackage);
204
    if (LSMversion[0] == 0) printf("Warning: no LSM version for %s.zip!\n", LSMpackage);
207
    if (LSMdescription[0] == 0) printf("Warning: no LSM description for %s.zip!\n", LSMpackage);
205
    if (LSMdescription[0] == 0) printf("Warning: no LSM description for %s.zip!\n", LSMpackage);
208
    fprintf(idx, "%s\t%s\t%s\t%u\n", LSMpackage, LSMversion, LSMdescription, bsum);
206
    fprintf(idx, "%s\t%s\t%s\t%u\n", LSMpackage, LSMversion, LSMdescription, bsum);
209
  }
207
  }
210
  fclose(idx);
208
  fclose(idx);
211
  printf("%d packages found.\n", LsmCount);
209
  printf("%d packages found.\n", LsmCount);
212
}
210
}
213
 
211
 
214
 
212
 
215
 
213
 
216
int main(int argc, char **argv) {
214
int main(int argc, char **argv) {
217
  char *repodir;
215
  char *repodir;
218
  char cmdbuff[256];
216
  char cmdbuff[256];
219
 
217
 
220
  puts("SvarDOS repository index generator ver " pVer);
218
  puts("SvarDOS repository index generator ver " pVer);
221
 
219
 
222
  if (argc != 2) {
220
  if (argc != 2) {
223
    puts("Usage: buildidx repodir");
221
    puts("Usage: buildidx repodir");
224
    return(1);
222
    return(1);
225
  }
223
  }
226
 
224
 
227
  if ((argv[1][0] == '?') || (argv[1][0] == '-') || (argv[1][0]) == 0) {
225
  if ((argv[1][0] == '?') || (argv[1][0] == '-') || (argv[1][0]) == 0) {
228
    puts("Usage: buildidx repodir");
226
    puts("Usage: buildidx repodir");
229
    return(1);
227
    return(1);
230
  }
228
  }
231
 
229
 
232
  repodir = argv[1];
230
  repodir = argv[1];
233
 
231
 
234
  printf("Will build index for %s directory.\n", repodir);
232
  printf("Will build index for %s directory.\n", repodir);
235
  puts("Recreating the appinfo directory...");
233
  puts("Recreating the appinfo directory...");
236
 
234
 
237
  system("rm -rf appinfo");
235
  system("rm -rf appinfo");
238
  system("mkdir appinfo");
236
  system("mkdir appinfo");
239
 
237
 
240
  puts("Populating appinfo with LSM files from archives...");
238
  puts("Populating appinfo with LSM files from archives...");
241
  snprintf(cmdbuff, sizeof(cmdbuff), "unzip -C -j -L -o '%s/*.zip' 'appinfo/*.lsm' -d appinfo", repodir);
239
  snprintf(cmdbuff, sizeof(cmdbuff), "unzip -C -j -L -o '%s/*.zip' 'appinfo/*.lsm' -d appinfo", repodir);
242
  system(cmdbuff);
240
  system(cmdbuff);
243
 
241
 
244
  puts("Generating the index file...");
242
  puts("Generating the index file...");
245
  GenIndexes(repodir);
243
  GenIndexes(repodir);
246
 
244
 
247
  system("rm -rf appinfo");
245
  system("rm -rf appinfo");
248
  return(0);
246
  return(0);
249
}
247
}
250
 
248