Subversion Repositories SvarDOS

Rev

Rev 314 | Only display areas with differences | Ignore whitespace | Details | Blame | Last modification | View Log | RSS feed

Rev 314 Rev 562
1
/*
1
/*
2
  SvarDOS repo index builder
2
  SvarDOS repo index builder
3
  Copyright (C) Mateusz Viste 2012-2021
3
  Copyright (C) Mateusz Viste 2012-2022
4

4

5
  buildidx computes an index tsv file for the SvarDOS repository.
5
  buildidx computes an index tsv file for the SvarDOS repository.
6
  it must be executed pointing to a directory that stores packages (zip)
6
  it must be executed pointing to a directory that stores packages (zip)
7
  files. buildidx will generate the index file and save it into the package
7
  files. buildidx will generate the index file and save it into the package
8
  repository.
8
  repository.
9

9

10
  13 feb 2021: 'title' LSM field is no longer looked after
10
  13 feb 2021: 'title' LSM field is no longer looked after
11
  11 feb 2021: lsm headers are no longer checked, so it is compatible with the simpler lsm format used by SvarDOS
11
  11 feb 2021: lsm headers are no longer checked, so it is compatible with the simpler lsm format used by SvarDOS
12
  13 jan 2021: removed the identification line, changed CRC32 to bsum, not creating the listing.txt file and stopped compressing index
12
  13 jan 2021: removed the identification line, changed CRC32 to bsum, not creating the listing.txt file and stopped compressing index
13
  23 apr 2017: uncompressed index is no longer created, added CRC32 of zib (bin only) files, if present
13
  23 apr 2017: uncompressed index is no longer created, added CRC32 of zib (bin only) files, if present
14
  28 aug 2016: listing.txt is always written inside the repo dir (instead of inside current dir)
14
  28 aug 2016: listing.txt is always written inside the repo dir (instead of inside current dir)
15
  27 aug 2016: accepting full paths to repos (starting with /...)
15
  27 aug 2016: accepting full paths to repos (starting with /...)
16
  07 dec 2013: rewritten buildidx in ANSI C89
16
  07 dec 2013: rewritten buildidx in ANSI C89
17
  19 aug 2013: add a compressed version of the index file to repos (index.gz)
17
  19 aug 2013: add a compressed version of the index file to repos (index.gz)
18
  22 jul 2013: creating a listing.txt file with list of packages
18
  22 jul 2013: creating a listing.txt file with list of packages
19
  18 jul 2013: writing the number of packaged into the first line of the lst file
19
  18 jul 2013: writing the number of packaged into the first line of the lst file
20
  11 jul 2013: added a switch to 7za to make it case insensitive when extracting lsm files
20
  11 jul 2013: added a switch to 7za to make it case insensitive when extracting lsm files
21
  10 jul 2013: changed unzip calls to 7za (to handle cases when appinfo is compressed with lzma)
21
  10 jul 2013: changed unzip calls to 7za (to handle cases when appinfo is compressed with lzma)
22
  04 feb 2013: added CRC32 support
22
  04 feb 2013: added CRC32 support
23
  22 sep 2012: forked 1st version from FDUPDATE builder
23
  22 sep 2012: forked 1st version from FDUPDATE builder
24
*/
24
*/
25
 
25
 
26
#include <errno.h>
26
#include <errno.h>
27
#include <stdio.h>   /* fopen, fclose... */
27
#include <stdio.h>   /* fopen, fclose... */
28
#include <stdint.h>
28
#include <stdint.h>
29
#include <stdlib.h>  /* system() */
29
#include <stdlib.h>  /* system() */
30
#include <string.h>  /* strcasecmp() */
30
#include <string.h>  /* strcasecmp() */
31
#include <time.h>    /* time(), ctime() */
31
#include <time.h>    /* time(), ctime() */
32
#include <unistd.h>  /* read() */
32
#include <unistd.h>  /* read() */
33
#include <dirent.h>
33
#include <dirent.h>
34
#include <sys/types.h>
34
#include <sys/types.h>
35
 
35
 
36
#define pVer "2021-02-11"
36
#define pVer "2022-01-20"
37
 
37
 
38
 
38
 
39
/* computes the BSD sum of a file and returns it. returns 0 on error. */
39
/* computes the BSD sum of a file and returns it. returns 0 on error. */
40
static uint16_t file2bsum(const char *filename) {
40
static uint16_t file2bsum(const char *filename) {
41
  uint16_t result = 0;
41
  uint16_t result = 0;
42
  unsigned char buff[1024 * 1024];
42
  unsigned char buff[1024 * 1024];
43
  size_t i, buffread;
43
  size_t i, buffread;
44
  FILE *fd;
44
  FILE *fd;
45
  fd = fopen(filename, "rb");
45
  fd = fopen(filename, "rb");
46
  if (fd == NULL) return(0);
46
  if (fd == NULL) return(0);
47
  while ((buffread = fread(buff, 1, sizeof(buff), fd)) > 0) {
47
  while ((buffread = fread(buff, 1, sizeof(buff), fd)) > 0) {
48
    for (i = 0; i < buffread; i++) {
48
    for (i = 0; i < buffread; i++) {
49
      /* rotr */
49
      /* rotr */
50
      result = (result >> 1) | (result << 15);
50
      result = (result >> 1) | (result << 15);
51
      /* */
51
      /* */
52
      result += buff[i];
52
      result += buff[i];
53
    }
53
    }
54
  }
54
  }
55
  fclose(fd);
55
  fclose(fd);
56
  return(result);
56
  return(result);
57
}
57
}
58
 
58
 
59
 
59
 
60
static void trim(char *str) {
60
static void trim(char *str) {
61
  int x, y, firstchar = -1, lastchar = -1;
61
  int x, y, firstchar = -1, lastchar = -1;
62
  for (x = 0; str[x] != 0; x++) {
62
  for (x = 0; str[x] != 0; x++) {
63
    switch (str[x]) {
63
    switch (str[x]) {
64
      case ' ':
64
      case ' ':
65
      case '\t':
65
      case '\t':
66
      case '\n':
66
      case '\n':
67
      case '\r':
67
      case '\r':
68
        break;
68
        break;
69
      default:
69
      default:
70
        if (firstchar < 0) firstchar = x;
70
        if (firstchar < 0) firstchar = x;
71
        lastchar = x;
71
        lastchar = x;
72
        break;
72
        break;
73
    }
73
    }
74
  }
74
  }
75
  str[lastchar + 1] = 0; /* right trim */
75
  str[lastchar + 1] = 0; /* right trim */
76
  if (firstchar > 0) { /* left trim (shift to the left ) */
76
  if (firstchar > 0) { /* left trim (shift to the left ) */
77
    y = 0;
77
    y = 0;
78
    for (x = firstchar; str[x] != 0; x++) str[y++] = str[x];
78
    for (x = firstchar; str[x] != 0; x++) str[y++] = str[x];
79
    str[y] = 0;
79
    str[y] = 0;
80
  }
80
  }
81
}
81
}
82
 
82
 
83
 
83
 
84
/* reads a line from a file descriptor, and writes it to *line, the *line array is filled no more than maxlen bytes. returns the number of byte read on success, or a negative value on failure (reaching EOF is considered an error condition) */
84
/* reads a line from a file descriptor, and writes it to *line, the *line array is filled no more than maxlen bytes. returns the number of byte read on success, or a negative value on failure (reaching EOF is considered an error condition) */
85
static int readline_fromfile(FILE *fd, char *line, int maxlen) {
85
static int readline_fromfile(FILE *fd, char *line, int maxlen) {
86
  int bytebuff, linelen = 0;
86
  int bytebuff, linelen = 0;
87
  for (;;) {
87
  for (;;) {
88
    bytebuff = fgetc(fd);
88
    bytebuff = fgetc(fd);
89
    if (bytebuff == EOF) {
89
    if (bytebuff == EOF) {
90
      line[linelen] = 0;
90
      line[linelen] = 0;
91
      if (linelen == 0) return(-1);
91
      if (linelen == 0) return(-1);
92
      return(linelen);
92
      return(linelen);
93
    }
93
    }
94
    if (bytebuff < 0) return(-1);
94
    if (bytebuff < 0) return(-1);
95
    if (bytebuff == '\r') continue; /* ignore CR */
95
    if (bytebuff == '\r') continue; /* ignore CR */
96
    if (bytebuff == '\n') {
96
    if (bytebuff == '\n') {
97
      line[linelen] = 0;
97
      line[linelen] = 0;
98
      return(linelen);
98
      return(linelen);
99
    }
99
    }
100
    if (linelen < maxlen) line[linelen++] = bytebuff;
100
    if (linelen < maxlen) line[linelen++] = bytebuff;
101
  }
101
  }
102
}
102
}
103
 
103
 
104
static int readlsm(const char *filename, char *version, char *description) {
104
static int readlsm(const char *filename, char *version, char *description) {
105
  char linebuff[1024];
105
  char linebuff[1024];
106
  char *valuestr;
106
  char *valuestr;
107
  int x;
107
  int x;
108
  FILE *fd;
108
  FILE *fd;
109
  /* reset fields to be read to empty values */
109
  /* reset fields to be read to empty values */
110
  version[0] = 0;
110
  version[0] = 0;
111
  description[0] = 0;
111
  description[0] = 0;
112
  /* open the file */
112
  /* open the file */
113
  fd = fopen(filename, "rb");
113
  fd = fopen(filename, "rb");
114
  if (fd == NULL) return(-1);
114
  if (fd == NULL) return(-1);
115
  /* read the LSM file line by line */
115
  /* read the LSM file line by line */
116
  while (readline_fromfile(fd, linebuff, 127) >= 0) {
116
  while (readline_fromfile(fd, linebuff, 127) >= 0) {
117
    for (x = 0;; x++) {
117
    for (x = 0;; x++) {
118
      if (linebuff[x] == 0) {
118
      if (linebuff[x] == 0) {
119
        x = -1;
119
        x = -1;
120
        break;
120
        break;
121
      } else if (linebuff[x] == ':') {
121
      } else if (linebuff[x] == ':') {
122
        break;
122
        break;
123
      }
123
      }
124
    }
124
    }
125
    if (x > 0) {
125
    if (x > 0) {
126
      linebuff[x] = 0;
126
      linebuff[x] = 0;
127
      valuestr = linebuff + x + 1;
127
      valuestr = linebuff + x + 1;
128
      trim(linebuff);
128
      trim(linebuff);
129
      trim(valuestr);
129
      trim(valuestr);
130
      if (strcasecmp(linebuff, "version") == 0) {
130
      if (strcasecmp(linebuff, "version") == 0) {
131
        sprintf(version, "%s", valuestr);
131
        sprintf(version, "%s", valuestr);
132
      } else if (strcasecmp(linebuff, "description") == 0) {
132
      } else if (strcasecmp(linebuff, "description") == 0) {
133
        sprintf(description, "%s", valuestr);
133
        sprintf(description, "%s", valuestr);
134
      }
134
      }
135
    }
135
    }
136
  }
136
  }
137
  fclose(fd);
137
  fclose(fd);
138
  return(0);
138
  return(0);
139
}
139
}
140
 
140
 
141
 
141
 
142
static int cmpstring(const void *p1, const void *p2) {
142
static int cmpstring(const void *p1, const void *p2) {
143
    /* The actual arguments to this function are "pointers to
143
    /* The actual arguments to this function are "pointers to
144
       pointers to char", but strcmp(3) arguments are "pointers
144
       pointers to char", but strcmp(3) arguments are "pointers
145
       to char", hence the following cast plus dereference */
145
       to char", hence the following cast plus dereference */
146
   return(strcasecmp(* (char * const *) p1, * (char * const *) p2));
146
   return(strcasecmp(* (char * const *) p1, * (char * const *) p2));
147
}
147
}
148
 
148
 
149
 
149
 
150
static void GenIndexes(const char *repodir) {
150
static void GenIndexes(const char *repodir) {
151
  char *LsmFileList[4096]; /* TODO dynamic sizing, otherwise will crash if there are 4096+ packages */
151
  char *LsmFileList[4096]; /* TODO dynamic sizing, otherwise will crash if there are 4096+ packages */
152
  char tmpbuf[64];
152
  char tmpbuf[64];
153
  char *LsmFile, LSMpackage[64], LSMversion[128], LSMdescription[1024];
153
  char *LsmFile, LSMpackage[64], LSMversion[128], LSMdescription[1024];
154
  int LsmCount = 0, x;
154
  int LsmCount = 0, x;
155
  FILE *idx;
155
  FILE *idx;
156
  DIR *dir;
156
  DIR *dir;
157
  struct dirent *diritem;
157
  struct dirent *diritem;
158
 
158
 
159
  dir = opendir("appinfo");
159
  dir = opendir("appinfo");
160
  if (dir == NULL) {
160
  if (dir == NULL) {
161
    printf("ERROR: Unable to open directory '%s' (%s)\n", repodir, strerror(errno));
161
    printf("ERROR: Unable to open directory '%s' (%s)\n", repodir, strerror(errno));
162
    return;
162
    return;
163
  }
163
  }
164
 
164
 
165
  /* load the content of the appinfo directory */
165
  /* load the content of the appinfo directory */
166
  while ((diritem = readdir(dir)) != NULL) {
166
  while ((diritem = readdir(dir)) != NULL) {
167
    if (strstr(diritem->d_name, ".lsm") == NULL) continue; /* skip files that do not contain '.lsm' */
167
    if (strstr(diritem->d_name, ".lsm") == NULL) continue; /* skip files that do not contain '.lsm' */
168
    LsmFileList[LsmCount] = strdup(diritem->d_name);
168
    LsmFileList[LsmCount] = strdup(diritem->d_name);
169
    /* printf("Loaded LSM file: %s\n", LsmFileList[LsmCount]); */
169
    /* printf("Loaded LSM file: %s\n", LsmFileList[LsmCount]); */
170
    LsmCount += 1;
170
    LsmCount += 1;
171
  }
171
  }
172
 
172
 
173
  closedir(dir);
173
  closedir(dir);
174
 
174
 
175
  printf("Found %d LSM files\n", LsmCount);
175
  printf("Found %d LSM files\n", LsmCount);
176
 
176
 
177
  /* sort the entries */
177
  /* sort the entries */
178
  qsort(&LsmFileList[0], LsmCount, sizeof(char *), cmpstring);
178
  qsort(&LsmFileList[0], LsmCount, sizeof(char *), cmpstring);
179
 
179
 
180
  /* Create the index file */
180
  /* Create the index file */
181
  sprintf(tmpbuf, "%s/index.tsv", repodir);
181
  sprintf(tmpbuf, "%s/index.tsv", repodir);
182
  idx = fopen(tmpbuf, "wb");
182
  idx = fopen(tmpbuf, "wb");
183
 
183
 
184
  /* Read every LSM */
184
  /* Read every LSM */
185
  for (x = 0; x < LsmCount; x++) {
185
  for (x = 0; x < LsmCount; x++) {
186
    uint16_t bsum;
186
    uint16_t bsum;
187
    LsmFile = LsmFileList[x];
187
    LsmFile = LsmFileList[x];
188
    sprintf(LSMpackage, "%s", LsmFile);
188
    sprintf(LSMpackage, "%s", LsmFile);
189
    LSMpackage[strlen(LSMpackage) - 4] = 0;
189
    LSMpackage[strlen(LSMpackage) - 4] = 0;
190
 
190
 
191
    /* compute the BSD sum of the zip package */
191
    /* compute the BSD sum of the zip package */
192
    sprintf(tmpbuf, "%s/%s.zip", repodir, LSMpackage);
192
    sprintf(tmpbuf, "%s/%s.zip", repodir, LSMpackage);
193
    bsum = file2bsum(tmpbuf);
193
    bsum = file2bsum(tmpbuf);
194
 
194
 
195
    printf("Processing %s... BSUM %04X\n", LsmFile, bsum);
195
    printf("Processing %s... BSUM %04X\n", LsmFile, bsum);
196
 
196
 
197
    sprintf(tmpbuf, "appinfo/%s", LsmFile);
197
    sprintf(tmpbuf, "appinfo/%s", LsmFile);
198
    readlsm(tmpbuf, LSMversion, LSMdescription);
198
    readlsm(tmpbuf, LSMversion, LSMdescription);
199
 
199
 
200
    if (strlen(LSMpackage) > 8) printf("Warning: %s.zip is not in 8.3 format!\n", LSMpackage);
200
    if (strlen(LSMpackage) > 8) printf("Warning: %s.zip is not in 8.3 format!\n", LSMpackage);
201
    if (LSMversion[0] == 0) printf("Warning: no LSM version for %s.zip!\n", LSMpackage);
201
    if (LSMversion[0] == 0) printf("Warning: no LSM version for %s.zip!\n", LSMpackage);
202
    if (LSMdescription[0] == 0) printf("Warning: no LSM description for %s.zip!\n", LSMpackage);
202
    if (LSMdescription[0] == 0) printf("Warning: no LSM description for %s.zip!\n", LSMpackage);
203
    fprintf(idx, "%s\t%s\t%s\t%u\n", LSMpackage, LSMversion, LSMdescription, bsum);
203
    fprintf(idx, "%s\t%s\t%s\t%u\n", LSMpackage, LSMversion, LSMdescription, bsum);
204
  }
204
  }
205
  fclose(idx);
205
  fclose(idx);
206
  printf("%d packages found.\n", LsmCount);
206
  printf("%d packages found.\n", LsmCount);
207
}
207
}
208
 
208
 
209
 
209
 
210
 
210
 
211
int main(int argc, char **argv) {
211
int main(int argc, char **argv) {
212
  char *repodir;
212
  char *repodir;
213
  char cmdbuff[256];
213
  char cmdbuff[256];
214
 
214
 
215
  puts("SvarDOS repository index generator ver " pVer);
215
  puts("SvarDOS repository index generator ver " pVer);
216
 
216
 
217
  if (argc != 2) {
217
  if (argc != 2) {
218
    puts("Usage: buildidx repodir");
218
    puts("Usage: buildidx repodir");
219
    return(1);
219
    return(1);
220
  }
220
  }
221
 
221
 
222
  if ((argv[1][0] == '?') || (argv[1][0] == '-') || (argv[1][0]) == 0) {
222
  if ((argv[1][0] == '?') || (argv[1][0] == '-') || (argv[1][0]) == 0) {
223
    puts("Usage: buildidx repodir");
223
    puts("Usage: buildidx repodir");
224
    return(1);
224
    return(1);
225
  }
225
  }
226
 
226
 
227
  repodir = argv[1];
227
  repodir = argv[1];
228
 
228
 
229
  printf("Will build index for %s directory.\n", repodir);
229
  printf("Will build index for %s directory.\n", repodir);
230
  puts("Recreating the appinfo directory...");
230
  puts("Recreating the appinfo directory...");
231
 
231
 
232
  system("rm -rf appinfo");
232
  system("rm -rf appinfo");
233
  system("mkdir appinfo");
233
  system("mkdir appinfo");
234
 
234
 
235
  puts("Populating appinfo with LSM files from archives...");
235
  puts("Populating appinfo with LSM files from archives...");
236
  snprintf(cmdbuff, sizeof(cmdbuff), "unzip -C -j -L -o '%s/*.zip' 'appinfo/*.lsm' -d appinfo", repodir);
236
  snprintf(cmdbuff, sizeof(cmdbuff), "unzip -C -j -L -o '%s/*.zip' 'appinfo/*.lsm' -d appinfo", repodir);
237
  system(cmdbuff);
237
  system(cmdbuff);
238
 
238
 
239
  puts("Generating the index file...");
239
  puts("Generating the index file...");
240
  GenIndexes(repodir);
240
  GenIndexes(repodir);
241
 
241
 
242
  system("rm -rf appinfo");
242
  system("rm -rf appinfo");
243
  return(0);
243
  return(0);
244
}
244
}
245
 
245