Subversion Repositories SvarDOS

Rev

Rev 292 | Go to most recent revision | Only display areas with differences | Ignore whitespace | Details | Blame | Last modification | View Log | RSS feed

Rev 292 Rev 314
1
/*
1
/*
2
  SvarDOS repo index builder
2
  SvarDOS repo index builder
3
  Copyright (C) Mateusz Viste 2012-2021
3
  Copyright (C) Mateusz Viste 2012-2021
4

4

5
  buildidx computes an index tsv file for the SvarDOS repository.
5
  buildidx computes an index tsv file for the SvarDOS repository.
6
  it must be executed pointing to a directory that stores packages (zip)
6
  it must be executed pointing to a directory that stores packages (zip)
7
  files. buildidx will generate the index file and save it into the package
7
  files. buildidx will generate the index file and save it into the package
8
  repository.
8
  repository.
9

9

-
 
10
  13 feb 2021: 'title' LSM field is no longer looked after
10
  11 feb 2021: lsm headers are no longer checked, so it is compatible with the simpler lsm format used by SvarDOS
11
  11 feb 2021: lsm headers are no longer checked, so it is compatible with the simpler lsm format used by SvarDOS
11
  13 jan 2021: removed the identification line, changed CRC32 to bsum, not creating the listing.txt file and stopped compressing index
12
  13 jan 2021: removed the identification line, changed CRC32 to bsum, not creating the listing.txt file and stopped compressing index
12
  23 apr 2017: uncompressed index is no longer created, added CRC32 of zib (bin only) files, if present
13
  23 apr 2017: uncompressed index is no longer created, added CRC32 of zib (bin only) files, if present
13
  28 aug 2016: listing.txt is always written inside the repo dir (instead of inside current dir)
14
  28 aug 2016: listing.txt is always written inside the repo dir (instead of inside current dir)
14
  27 aug 2016: accepting full paths to repos (starting with /...)
15
  27 aug 2016: accepting full paths to repos (starting with /...)
15
  07 dec 2013: rewritten buildidx in ANSI C89
16
  07 dec 2013: rewritten buildidx in ANSI C89
16
  19 aug 2013: add a compressed version of the index file to repos (index.gz)
17
  19 aug 2013: add a compressed version of the index file to repos (index.gz)
17
  22 jul 2013: creating a listing.txt file with list of packages
18
  22 jul 2013: creating a listing.txt file with list of packages
18
  18 jul 2013: writing the number of packaged into the first line of the lst file
19
  18 jul 2013: writing the number of packaged into the first line of the lst file
19
  11 jul 2013: added a switch to 7za to make it case insensitive when extracting lsm files
20
  11 jul 2013: added a switch to 7za to make it case insensitive when extracting lsm files
20
  10 jul 2013: changed unzip calls to 7za (to handle cases when appinfo is compressed with lzma)
21
  10 jul 2013: changed unzip calls to 7za (to handle cases when appinfo is compressed with lzma)
21
  04 feb 2013: added CRC32 support
22
  04 feb 2013: added CRC32 support
22
  22 sep 2012: forked 1st version from FDUPDATE builder
23
  22 sep 2012: forked 1st version from FDUPDATE builder
23
*/
24
*/
24
 
25
 
25
#include <errno.h>
26
#include <errno.h>
26
#include <stdio.h>   /* fopen, fclose... */
27
#include <stdio.h>   /* fopen, fclose... */
27
#include <stdint.h>
28
#include <stdint.h>
28
#include <stdlib.h>  /* system() */
29
#include <stdlib.h>  /* system() */
29
#include <string.h>  /* strcasecmp() */
30
#include <string.h>  /* strcasecmp() */
30
#include <time.h>    /* time(), ctime() */
31
#include <time.h>    /* time(), ctime() */
31
#include <unistd.h>  /* read() */
32
#include <unistd.h>  /* read() */
32
#include <dirent.h>
33
#include <dirent.h>
33
#include <sys/types.h>
34
#include <sys/types.h>
34
 
35
 
35
#define pVer "2021-02-11"
36
#define pVer "2021-02-11"
36
 
37
 
37
 
38
 
38
/* computes the BSD sum of a file and returns it. returns 0 on error. */
39
/* computes the BSD sum of a file and returns it. returns 0 on error. */
39
static uint16_t file2bsum(const char *filename) {
40
static uint16_t file2bsum(const char *filename) {
40
  uint16_t result = 0;
41
  uint16_t result = 0;
41
  unsigned char buff[1024 * 1024];
42
  unsigned char buff[1024 * 1024];
42
  size_t i, buffread;
43
  size_t i, buffread;
43
  FILE *fd;
44
  FILE *fd;
44
  fd = fopen(filename, "rb");
45
  fd = fopen(filename, "rb");
45
  if (fd == NULL) return(0);
46
  if (fd == NULL) return(0);
46
  while ((buffread = fread(buff, 1, sizeof(buff), fd)) > 0) {
47
  while ((buffread = fread(buff, 1, sizeof(buff), fd)) > 0) {
47
    for (i = 0; i < buffread; i++) {
48
    for (i = 0; i < buffread; i++) {
48
      /* rotr */
49
      /* rotr */
49
      result = (result >> 1) | (result << 15);
50
      result = (result >> 1) | (result << 15);
50
      /* */
51
      /* */
51
      result += buff[i];
52
      result += buff[i];
52
    }
53
    }
53
  }
54
  }
54
  fclose(fd);
55
  fclose(fd);
55
  return(result);
56
  return(result);
56
}
57
}
57
 
58
 
58
 
59
 
59
static void trim(char *str) {
60
static void trim(char *str) {
60
  int x, y, firstchar = -1, lastchar = -1;
61
  int x, y, firstchar = -1, lastchar = -1;
61
  for (x = 0; str[x] != 0; x++) {
62
  for (x = 0; str[x] != 0; x++) {
62
    switch (str[x]) {
63
    switch (str[x]) {
63
      case ' ':
64
      case ' ':
64
      case '\t':
65
      case '\t':
65
      case '\n':
66
      case '\n':
66
      case '\r':
67
      case '\r':
67
        break;
68
        break;
68
      default:
69
      default:
69
        if (firstchar < 0) firstchar = x;
70
        if (firstchar < 0) firstchar = x;
70
        lastchar = x;
71
        lastchar = x;
71
        break;
72
        break;
72
    }
73
    }
73
  }
74
  }
74
  str[lastchar + 1] = 0; /* right trim */
75
  str[lastchar + 1] = 0; /* right trim */
75
  if (firstchar > 0) { /* left trim (shift to the left ) */
76
  if (firstchar > 0) { /* left trim (shift to the left ) */
76
    y = 0;
77
    y = 0;
77
    for (x = firstchar; str[x] != 0; x++) str[y++] = str[x];
78
    for (x = firstchar; str[x] != 0; x++) str[y++] = str[x];
78
    str[y] = 0;
79
    str[y] = 0;
79
  }
80
  }
80
}
81
}
81
 
82
 
82
 
83
 
83
/* reads a line from a file descriptor, and writes it to *line, the *line array is filled no more than maxlen bytes. returns the number of byte read on success, or a negative value on failure (reaching EOF is considered an error condition) */
84
/* reads a line from a file descriptor, and writes it to *line, the *line array is filled no more than maxlen bytes. returns the number of byte read on success, or a negative value on failure (reaching EOF is considered an error condition) */
84
static int readline_fromfile(FILE *fd, char *line, int maxlen) {
85
static int readline_fromfile(FILE *fd, char *line, int maxlen) {
85
  int bytebuff, linelen = 0;
86
  int bytebuff, linelen = 0;
86
  for (;;) {
87
  for (;;) {
87
    bytebuff = fgetc(fd);
88
    bytebuff = fgetc(fd);
88
    if (bytebuff == EOF) {
89
    if (bytebuff == EOF) {
89
      line[linelen] = 0;
90
      line[linelen] = 0;
90
      if (linelen == 0) return(-1);
91
      if (linelen == 0) return(-1);
91
      return(linelen);
92
      return(linelen);
92
    }
93
    }
93
    if (bytebuff < 0) return(-1);
94
    if (bytebuff < 0) return(-1);
94
    if (bytebuff == '\r') continue; /* ignore CR */
95
    if (bytebuff == '\r') continue; /* ignore CR */
95
    if (bytebuff == '\n') {
96
    if (bytebuff == '\n') {
96
      line[linelen] = 0;
97
      line[linelen] = 0;
97
      return(linelen);
98
      return(linelen);
98
    }
99
    }
99
    if (linelen < maxlen) line[linelen++] = bytebuff;
100
    if (linelen < maxlen) line[linelen++] = bytebuff;
100
  }
101
  }
101
}
102
}
102
 
103
 
103
static int readlsm(const char *filename, char *version, char *title, char *description) {
104
static int readlsm(const char *filename, char *version, char *description) {
104
  char linebuff[1024];
105
  char linebuff[1024];
105
  char *valuestr;
106
  char *valuestr;
106
  int x;
107
  int x;
107
  FILE *fd;
108
  FILE *fd;
108
  /* reset fields to be read to empty values */
109
  /* reset fields to be read to empty values */
109
  version[0] = 0;
110
  version[0] = 0;
110
  description[0] = 0;
111
  description[0] = 0;
111
  title[0] = 0;
-
 
112
  /* open the file */
112
  /* open the file */
113
  fd = fopen(filename, "rb");
113
  fd = fopen(filename, "rb");
114
  if (fd == NULL) return(-1);
114
  if (fd == NULL) return(-1);
115
  /* read the LSM file line by line */
115
  /* read the LSM file line by line */
116
  while (readline_fromfile(fd, linebuff, 127) >= 0) {
116
  while (readline_fromfile(fd, linebuff, 127) >= 0) {
117
    for (x = 0;; x++) {
117
    for (x = 0;; x++) {
118
      if (linebuff[x] == 0) {
118
      if (linebuff[x] == 0) {
119
        x = -1;
119
        x = -1;
120
        break;
120
        break;
121
      } else if (linebuff[x] == ':') {
121
      } else if (linebuff[x] == ':') {
122
        break;
122
        break;
123
      }
123
      }
124
    }
124
    }
125
    if (x > 0) {
125
    if (x > 0) {
126
      linebuff[x] = 0;
126
      linebuff[x] = 0;
127
      valuestr = linebuff + x + 1;
127
      valuestr = linebuff + x + 1;
128
      trim(linebuff);
128
      trim(linebuff);
129
      trim(valuestr);
129
      trim(valuestr);
130
      if (strcasecmp(linebuff, "version") == 0) {
130
      if (strcasecmp(linebuff, "version") == 0) {
131
          sprintf(version, "%s", valuestr);
131
        sprintf(version, "%s", valuestr);
132
        } else if (strcasecmp(linebuff, "title") == 0) {
-
 
133
          sprintf(title, "%s", valuestr);
-
 
134
        } else if (strcasecmp(linebuff, "description") == 0) {
132
      } else if (strcasecmp(linebuff, "description") == 0) {
135
          sprintf(description, "%s", valuestr);
133
        sprintf(description, "%s", valuestr);
136
      }
134
      }
137
    }
135
    }
138
  }
136
  }
139
  fclose(fd);
137
  fclose(fd);
140
  return(0);
138
  return(0);
141
}
139
}
142
 
140
 
143
 
141
 
144
static int cmpstring(const void *p1, const void *p2) {
142
static int cmpstring(const void *p1, const void *p2) {
145
    /* The actual arguments to this function are "pointers to
143
    /* The actual arguments to this function are "pointers to
146
       pointers to char", but strcmp(3) arguments are "pointers
144
       pointers to char", but strcmp(3) arguments are "pointers
147
       to char", hence the following cast plus dereference */
145
       to char", hence the following cast plus dereference */
148
   return(strcasecmp(* (char * const *) p1, * (char * const *) p2));
146
   return(strcasecmp(* (char * const *) p1, * (char * const *) p2));
149
}
147
}
150
 
148
 
151
 
149
 
152
static void GenIndexes(const char *repodir) {
150
static void GenIndexes(const char *repodir) {
153
  char *LsmFileList[4096];
151
  char *LsmFileList[4096]; /* TODO dynamic sizing, otherwise will crash if there are 4096+ packages */
154
  char tmpbuf[64];
152
  char tmpbuf[64];
155
  char *LsmFile, LSMpackage[64], LSMtitle[128], LSMversion[128], LSMdescription[1024];
153
  char *LsmFile, LSMpackage[64], LSMversion[128], LSMdescription[1024];
156
  int LsmCount = 0, x;
154
  int LsmCount = 0, x;
157
  FILE *idx;
155
  FILE *idx;
158
  DIR *dir;
156
  DIR *dir;
159
  struct dirent *diritem;
157
  struct dirent *diritem;
160
 
158
 
161
  dir = opendir("appinfo");
159
  dir = opendir("appinfo");
162
  if (dir == NULL) {
160
  if (dir == NULL) {
163
    printf("ERROR: Unable to open directory '%s' (%s)\n", repodir, strerror(errno));
161
    printf("ERROR: Unable to open directory '%s' (%s)\n", repodir, strerror(errno));
164
    return;
162
    return;
165
  }
163
  }
166
 
164
 
167
  /* load the content of the appinfo directory */
165
  /* load the content of the appinfo directory */
168
  while ((diritem = readdir(dir)) != NULL) {
166
  while ((diritem = readdir(dir)) != NULL) {
169
    if (strstr(diritem->d_name, ".lsm") == NULL) continue; /* skip files that do not contain '.lsm' */
167
    if (strstr(diritem->d_name, ".lsm") == NULL) continue; /* skip files that do not contain '.lsm' */
170
    LsmFileList[LsmCount] = strdup(diritem->d_name);
168
    LsmFileList[LsmCount] = strdup(diritem->d_name);
171
    /* printf("Loaded LSM file: %s\n", LsmFileList[LsmCount]); */
169
    /* printf("Loaded LSM file: %s\n", LsmFileList[LsmCount]); */
172
    LsmCount += 1;
170
    LsmCount += 1;
173
  }
171
  }
174
 
172
 
175
  closedir(dir);
173
  closedir(dir);
176
 
174
 
177
  printf("Found %d LSM files\n", LsmCount);
175
  printf("Found %d LSM files\n", LsmCount);
178
 
176
 
179
  /* sort the entries */
177
  /* sort the entries */
180
  qsort(&LsmFileList[0], LsmCount, sizeof(char *), cmpstring);
178
  qsort(&LsmFileList[0], LsmCount, sizeof(char *), cmpstring);
181
 
179
 
182
  /* Create the index file */
180
  /* Create the index file */
183
  sprintf(tmpbuf, "%s/index.tsv", repodir);
181
  sprintf(tmpbuf, "%s/index.tsv", repodir);
184
  idx = fopen(tmpbuf, "wb");
182
  idx = fopen(tmpbuf, "wb");
185
 
183
 
186
  /* Read every LSM */
184
  /* Read every LSM */
187
  for (x = 0; x < LsmCount; x++) {
185
  for (x = 0; x < LsmCount; x++) {
188
    uint16_t bsum;
186
    uint16_t bsum;
189
    LsmFile = LsmFileList[x];
187
    LsmFile = LsmFileList[x];
190
    sprintf(LSMpackage, "%s", LsmFile);
188
    sprintf(LSMpackage, "%s", LsmFile);
191
    LSMpackage[strlen(LSMpackage) - 4] = 0;
189
    LSMpackage[strlen(LSMpackage) - 4] = 0;
192
 
190
 
193
    /* compute the BSD sum of the zip package */
191
    /* compute the BSD sum of the zip package */
194
    sprintf(tmpbuf, "%s/%s.zip", repodir, LSMpackage);
192
    sprintf(tmpbuf, "%s/%s.zip", repodir, LSMpackage);
195
    bsum = file2bsum(tmpbuf);
193
    bsum = file2bsum(tmpbuf);
196
 
194
 
197
    printf("Processing %s... BSUM %04X\n", LsmFile, bsum);
195
    printf("Processing %s... BSUM %04X\n", LsmFile, bsum);
198
 
196
 
199
    sprintf(tmpbuf, "appinfo/%s", LsmFile);
197
    sprintf(tmpbuf, "appinfo/%s", LsmFile);
200
    readlsm(tmpbuf, LSMversion, LSMtitle, LSMdescription);
198
    readlsm(tmpbuf, LSMversion, LSMdescription);
201
 
199
 
202
    if (strlen(LSMpackage) > 8) printf("Warning: %s.zip is not in 8.3 format!\n", LSMpackage);
200
    if (strlen(LSMpackage) > 8) printf("Warning: %s.zip is not in 8.3 format!\n", LSMpackage);
203
    if (LSMtitle[0] == 0) printf("Warning: no LSM title for %s.zip\n", LSMpackage);
-
 
204
    if (LSMversion[0] == 0) printf("Warning: no LSM version for %s.zip!\n", LSMpackage);
201
    if (LSMversion[0] == 0) printf("Warning: no LSM version for %s.zip!\n", LSMpackage);
205
    if (LSMdescription[0] == 0) printf("Warning: no LSM description for %s.zip!\n", LSMpackage);
202
    if (LSMdescription[0] == 0) printf("Warning: no LSM description for %s.zip!\n", LSMpackage);
206
    fprintf(idx, "%s\t%s\t%s\t%u\n", LSMpackage, LSMversion, LSMdescription, bsum);
203
    fprintf(idx, "%s\t%s\t%s\t%u\n", LSMpackage, LSMversion, LSMdescription, bsum);
207
  }
204
  }
208
  fclose(idx);
205
  fclose(idx);
209
  printf("%d packages found.\n", LsmCount);
206
  printf("%d packages found.\n", LsmCount);
210
}
207
}
211
 
208
 
212
 
209
 
213
 
210
 
214
int main(int argc, char **argv) {
211
int main(int argc, char **argv) {
215
  char *repodir;
212
  char *repodir;
216
  char cmdbuff[256];
213
  char cmdbuff[256];
217
 
214
 
218
  puts("SvarDOS repository index generator ver " pVer);
215
  puts("SvarDOS repository index generator ver " pVer);
219
 
216
 
220
  if (argc != 2) {
217
  if (argc != 2) {
221
    puts("Usage: buildidx repodir");
218
    puts("Usage: buildidx repodir");
222
    return(1);
219
    return(1);
223
  }
220
  }
224
 
221
 
225
  if ((argv[1][0] == '?') || (argv[1][0] == '-') || (argv[1][0]) == 0) {
222
  if ((argv[1][0] == '?') || (argv[1][0] == '-') || (argv[1][0]) == 0) {
226
    puts("Usage: buildidx repodir");
223
    puts("Usage: buildidx repodir");
227
    return(1);
224
    return(1);
228
  }
225
  }
229
 
226
 
230
  repodir = argv[1];
227
  repodir = argv[1];
231
 
228
 
232
  printf("Will build index for %s directory.\n", repodir);
229
  printf("Will build index for %s directory.\n", repodir);
233
  puts("Recreating the appinfo directory...");
230
  puts("Recreating the appinfo directory...");
234
 
231
 
235
  system("rm -rf appinfo");
232
  system("rm -rf appinfo");
236
  system("mkdir appinfo");
233
  system("mkdir appinfo");
237
 
234
 
238
  puts("Populating appinfo with LSM files from archives...");
235
  puts("Populating appinfo with LSM files from archives...");
239
  snprintf(cmdbuff, sizeof(cmdbuff), "unzip -C -j -L -o '%s/*.zip' 'appinfo/*.lsm' -d appinfo", repodir);
236
  snprintf(cmdbuff, sizeof(cmdbuff), "unzip -C -j -L -o '%s/*.zip' 'appinfo/*.lsm' -d appinfo", repodir);
240
  system(cmdbuff);
237
  system(cmdbuff);
241
 
238
 
242
  puts("Generating the index file...");
239
  puts("Generating the index file...");
243
  GenIndexes(repodir);
240
  GenIndexes(repodir);
244
 
241
 
245
  system("rm -rf appinfo");
242
  system("rm -rf appinfo");
246
  return(0);
243
  return(0);
247
}
244
}
248
 
245