Subversion Repositories SvarDOS

Rev

Rev 185 | Go to most recent revision | Only display areas with differences | Ignore whitespace | Details | Blame | Last modification | View Log | RSS feed

Rev 185 Rev 186
1
/*
1
/*
2
  SvarDOS repo index builder
2
  SvarDOS repo index builder
3
  Copyright (C) Mateusz Viste 2012-2021
3
  Copyright (C) Mateusz Viste 2012-2021
4

4

5
  buildidx computes an index tsv file for the SvarDOS repository.
5
  buildidx computes an index tsv file for the SvarDOS repository.
6
  it must be executed pointing to a directory that stores packages (zip)
6
  it must be executed pointing to a directory that stores packages (zip)
7
  files. buildidx will generate the index file and save it into the package
7
  files. buildidx will generate the index file and save it into the package
8
  repository.
8
  repository.
9

9

10
  13 jan 2021: removed the identification line, changed CRC32 to bsum, not creating the listing.txt file and stopped compressing index
10
  13 jan 2021: removed the identification line, changed CRC32 to bsum, not creating the listing.txt file and stopped compressing index
11
  23 apr 2017: uncompressed index is no longer created, added CRC32 of zib (bin only) files, if present
11
  23 apr 2017: uncompressed index is no longer created, added CRC32 of zib (bin only) files, if present
12
  28 aug 2016: listing.txt is always written inside the repo dir (instead of inside current dir)
12
  28 aug 2016: listing.txt is always written inside the repo dir (instead of inside current dir)
13
  27 aug 2016: accepting full paths to repos (starting with /...)
13
  27 aug 2016: accepting full paths to repos (starting with /...)
14
  07 dec 2013: rewritten buildidx in ANSI C89
14
  07 dec 2013: rewritten buildidx in ANSI C89
15
  19 aug 2013: add a compressed version of the index file to repos (index.gz)
15
  19 aug 2013: add a compressed version of the index file to repos (index.gz)
16
  22 jul 2013: creating a listing.txt file with list of packages
16
  22 jul 2013: creating a listing.txt file with list of packages
17
  18 jul 2013: writing the number of packaged into the first line of the lst file
17
  18 jul 2013: writing the number of packaged into the first line of the lst file
18
  11 jul 2013: added a switch to 7za to make it case insensitive when extracting lsm files
18
  11 jul 2013: added a switch to 7za to make it case insensitive when extracting lsm files
19
  10 jul 2013: changed unzip calls to 7za (to handle cases when appinfo is compressed with lzma)
19
  10 jul 2013: changed unzip calls to 7za (to handle cases when appinfo is compressed with lzma)
20
  04 feb 2013: added CRC32 support
20
  04 feb 2013: added CRC32 support
21
  22 sep 2012: forked 1st version from FDUPDATE builder
21
  22 sep 2012: forked 1st version from FDUPDATE builder
22
*/
22
*/
23
 
23
 
24
#include <errno.h>
24
#include <errno.h>
25
#include <stdio.h>   /* fopen, fclose... */
25
#include <stdio.h>   /* fopen, fclose... */
26
#include <stdint.h>
26
#include <stdint.h>
27
#include <stdlib.h>  /* system() */
27
#include <stdlib.h>  /* system() */
28
#include <string.h>  /* strcasecmp() */
28
#include <string.h>  /* strcasecmp() */
29
#include <time.h>    /* time(), ctime() */
29
#include <time.h>    /* time(), ctime() */
30
#include <unistd.h>  /* read() */
30
#include <unistd.h>  /* read() */
31
#include <dirent.h>
31
#include <dirent.h>
32
#include <sys/types.h>
32
#include <sys/types.h>
33
 
33
 
34
#define pVer "2021-01-13"
34
#define pVer "2021-01-13"
35
 
35
 
36
 
36
 
37
/* computes the BSD sum of a file and returns it. returns 0 on error. */
37
/* computes the BSD sum of a file and returns it. returns 0 on error. */
38
static uint16_t file2bsum(char *filename) {
38
static uint16_t file2bsum(const char *filename) {
39
  uint16_t result = 0;
39
  uint16_t result = 0;
40
  unsigned char buff[1024 * 1024];
40
  unsigned char buff[1024 * 1024];
41
  size_t i, buffread;
41
  size_t i, buffread;
42
  FILE *fd;
42
  FILE *fd;
43
  fd = fopen(filename, "rb");
43
  fd = fopen(filename, "rb");
44
  if (fd == NULL) return(0);
44
  if (fd == NULL) return(0);
45
  while ((buffread = fread(buff, 1, sizeof(buff), fd)) > 0) {
45
  while ((buffread = fread(buff, 1, sizeof(buff), fd)) > 0) {
46
    for (i = 0; i < buffread; i++) {
46
    for (i = 0; i < buffread; i++) {
47
      /* rotr */
47
      /* rotr */
48
      result = (result >> 1) | (result << 15);
48
      result = (result >> 1) | (result << 15);
49
      /* */
49
      /* */
50
      result += buff[i];
50
      result += buff[i];
51
    }
51
    }
52
  }
52
  }
53
  fclose(fd);
53
  fclose(fd);
54
  return(result);
54
  return(result);
55
}
55
}
56
 
56
 
57
 
57
 
58
static void trim(char *str) {
58
static void trim(char *str) {
59
  int x, y, firstchar = -1, lastchar = -1;
59
  int x, y, firstchar = -1, lastchar = -1;
60
  for (x = 0; str[x] != 0; x++) {
60
  for (x = 0; str[x] != 0; x++) {
61
    switch (str[x]) {
61
    switch (str[x]) {
62
      case ' ':
62
      case ' ':
63
      case '\t':
63
      case '\t':
64
      case '\n':
64
      case '\n':
65
      case '\r':
65
      case '\r':
66
        break;
66
        break;
67
      default:
67
      default:
68
        if (firstchar < 0) firstchar = x;
68
        if (firstchar < 0) firstchar = x;
69
        lastchar = x;
69
        lastchar = x;
70
        break;
70
        break;
71
    }
71
    }
72
  }
72
  }
73
  str[lastchar + 1] = 0; /* right trim */
73
  str[lastchar + 1] = 0; /* right trim */
74
  if (firstchar > 0) { /* left trim (shift to the left ) */
74
  if (firstchar > 0) { /* left trim (shift to the left ) */
75
    y = 0;
75
    y = 0;
76
    for (x = firstchar; str[x] != 0; x++) str[y++] = str[x];
76
    for (x = firstchar; str[x] != 0; x++) str[y++] = str[x];
77
    str[y] = 0;
77
    str[y] = 0;
78
  }
78
  }
79
}
79
}
80
 
80
 
81
 
81
 
82
/* reads a line from a file descriptor, and writes it to *line, the *line array is filled no more than maxlen bytes. returns the number of byte read on success, or a negative value on failure (reaching EOF is considered an error condition) */
82
/* reads a line from a file descriptor, and writes it to *line, the *line array is filled no more than maxlen bytes. returns the number of byte read on success, or a negative value on failure (reaching EOF is considered an error condition) */
83
static int readline_fromfile(FILE *fd, char *line, int maxlen) {
83
static int readline_fromfile(FILE *fd, char *line, int maxlen) {
84
  int bytebuff, linelen = 0;
84
  int bytebuff, linelen = 0;
85
  for (;;) {
85
  for (;;) {
86
    bytebuff = fgetc(fd);
86
    bytebuff = fgetc(fd);
87
    if (bytebuff == EOF) {
87
    if (bytebuff == EOF) {
88
      line[linelen] = 0;
88
      line[linelen] = 0;
89
      if (linelen == 0) return(-1);
89
      if (linelen == 0) return(-1);
90
      return(linelen);
90
      return(linelen);
91
    }
91
    }
92
    if (bytebuff < 0) return(-1);
92
    if (bytebuff < 0) return(-1);
93
    if (bytebuff == '\r') continue; /* ignore CR */
93
    if (bytebuff == '\r') continue; /* ignore CR */
94
    if (bytebuff == '\n') {
94
    if (bytebuff == '\n') {
95
      line[linelen] = 0;
95
      line[linelen] = 0;
96
      return(linelen);
96
      return(linelen);
97
    }
97
    }
98
    if (linelen < maxlen) line[linelen++] = bytebuff;
98
    if (linelen < maxlen) line[linelen++] = bytebuff;
99
  }
99
  }
100
}
100
}
101
 
101
 
102
static int readlsm(char *filename, char *version, char *title, char *description) {
102
static int readlsm(const char *filename, char *version, char *title, char *description) {
103
  char linebuff[1024];
103
  char linebuff[1024];
104
  char *valuestr;
104
  char *valuestr;
105
  int x;
105
  int x;
106
  FILE *fd;
106
  FILE *fd;
107
  /* reset fields to be read to empty values */
107
  /* reset fields to be read to empty values */
108
  version[0] = 0;
108
  version[0] = 0;
109
  description[0] = 0;
109
  description[0] = 0;
110
  title[0] = 0;
110
  title[0] = 0;
111
  /* open the file */
111
  /* open the file */
112
  fd = fopen(filename, "rb");
112
  fd = fopen(filename, "rb");
113
  if (fd == NULL) return(-1);
113
  if (fd == NULL) return(-1);
114
  /* check the file's header */
114
  /* check the file's header */
115
  if (readline_fromfile(fd, linebuff, 64) < 0) return(-1);
115
  if (readline_fromfile(fd, linebuff, 64) < 0) return(-1);
116
  if (strcasecmp(linebuff, "begin3") != 0) return(-1);
116
  if (strcasecmp(linebuff, "begin3") != 0) return(-1);
117
  /* read the LSM file line by line */
117
  /* read the LSM file line by line */
118
  while (readline_fromfile(fd, linebuff, 127) >= 0) {
118
  while (readline_fromfile(fd, linebuff, 127) >= 0) {
119
    for (x = 0;; x++) {
119
    for (x = 0;; x++) {
120
      if (linebuff[x] == 0) {
120
      if (linebuff[x] == 0) {
121
        x = -1;
121
        x = -1;
122
        break;
122
        break;
123
      } else if (linebuff[x] == ':') {
123
      } else if (linebuff[x] == ':') {
124
        break;
124
        break;
125
      }
125
      }
126
    }
126
    }
127
    if (x > 0) {
127
    if (x > 0) {
128
      linebuff[x] = 0;
128
      linebuff[x] = 0;
129
      valuestr = linebuff + x + 1;
129
      valuestr = linebuff + x + 1;
130
      trim(linebuff);
130
      trim(linebuff);
131
      trim(valuestr);
131
      trim(valuestr);
132
      if (strcasecmp(linebuff, "version") == 0) {
132
      if (strcasecmp(linebuff, "version") == 0) {
133
          sprintf(version, "%s", valuestr);
133
          sprintf(version, "%s", valuestr);
134
        } else if (strcasecmp(linebuff, "title") == 0) {
134
        } else if (strcasecmp(linebuff, "title") == 0) {
135
          sprintf(title, "%s", valuestr);
135
          sprintf(title, "%s", valuestr);
136
        } else if (strcasecmp(linebuff, "description") == 0) {
136
        } else if (strcasecmp(linebuff, "description") == 0) {
137
          sprintf(description, "%s", valuestr);
137
          sprintf(description, "%s", valuestr);
138
      }
138
      }
139
    }
139
    }
140
  }
140
  }
141
  fclose(fd);
141
  fclose(fd);
142
  return(0);
142
  return(0);
143
}
143
}
144
 
144
 
145
 
145
 
146
static int cmpstring(const void *p1, const void *p2) {
146
static int cmpstring(const void *p1, const void *p2) {
147
    /* The actual arguments to this function are "pointers to
147
    /* The actual arguments to this function are "pointers to
148
       pointers to char", but strcmp(3) arguments are "pointers
148
       pointers to char", but strcmp(3) arguments are "pointers
149
       to char", hence the following cast plus dereference */
149
       to char", hence the following cast plus dereference */
150
   return(strcasecmp(* (char * const *) p1, * (char * const *) p2));
150
   return(strcasecmp(* (char * const *) p1, * (char * const *) p2));
151
}
151
}
152
 
152
 
153
 
153
 
154
static void GenIndexes(char *repodir) {
154
static void GenIndexes(const char *repodir) {
155
  char *LsmFileList[4096];
155
  char *LsmFileList[4096];
156
  char tmpbuf[64];
156
  char tmpbuf[64];
157
  char *LsmFile, LSMpackage[64], LSMtitle[128], LSMversion[128], LSMdescription[1024];
157
  char *LsmFile, LSMpackage[64], LSMtitle[128], LSMversion[128], LSMdescription[1024];
158
  int LsmCount = 0, x;
158
  int LsmCount = 0, x;
159
  FILE *idx;
159
  FILE *idx;
160
  DIR *dir;
160
  DIR *dir;
161
  struct dirent *diritem;
161
  struct dirent *diritem;
162
 
162
 
163
  dir = opendir("appinfo");
163
  dir = opendir("appinfo");
164
  if (dir == NULL) {
164
  if (dir == NULL) {
165
    printf("ERROR: Unable to open directory '%s' (%s)\n", repodir, strerror(errno));
165
    printf("ERROR: Unable to open directory '%s' (%s)\n", repodir, strerror(errno));
166
    return;
166
    return;
167
  }
167
  }
168
 
168
 
169
  /* load the content of the appinfo directory */
169
  /* load the content of the appinfo directory */
170
  while ((diritem = readdir(dir)) != NULL) {
170
  while ((diritem = readdir(dir)) != NULL) {
171
    if (strstr(diritem->d_name, ".lsm") == NULL) continue; /* skip files that do not contain '.lsm' */
171
    if (strstr(diritem->d_name, ".lsm") == NULL) continue; /* skip files that do not contain '.lsm' */
172
    LsmFileList[LsmCount] = strdup(diritem->d_name);
172
    LsmFileList[LsmCount] = strdup(diritem->d_name);
173
    /* printf("Loaded LSM file: %s\n", LsmFileList[LsmCount]); */
173
    /* printf("Loaded LSM file: %s\n", LsmFileList[LsmCount]); */
174
    LsmCount += 1;
174
    LsmCount += 1;
175
  }
175
  }
176
 
176
 
177
  closedir(dir);
177
  closedir(dir);
178
 
178
 
179
  printf("Found %d LSM files\n", LsmCount);
179
  printf("Found %d LSM files\n", LsmCount);
180
 
180
 
181
  /* sort the entries */
181
  /* sort the entries */
182
  qsort(&LsmFileList[0], LsmCount, sizeof(char *), cmpstring);
182
  qsort(&LsmFileList[0], LsmCount, sizeof(char *), cmpstring);
183
 
183
 
184
  /* Create the index file */
184
  /* Create the index file */
185
  sprintf(tmpbuf, "%s/index.tsv", repodir);
185
  sprintf(tmpbuf, "%s/index.tsv", repodir);
186
  idx = fopen(tmpbuf, "wb");
186
  idx = fopen(tmpbuf, "wb");
187
 
187
 
188
  /* Read every LSM */
188
  /* Read every LSM */
189
  for (x = 0; x < LsmCount; x++) {
189
  for (x = 0; x < LsmCount; x++) {
190
    uint16_t bsum;
190
    uint16_t bsum;
191
    LsmFile = LsmFileList[x];
191
    LsmFile = LsmFileList[x];
192
    sprintf(LSMpackage, "%s", LsmFile);
192
    sprintf(LSMpackage, "%s", LsmFile);
193
    LSMpackage[strlen(LSMpackage) - 4] = 0;
193
    LSMpackage[strlen(LSMpackage) - 4] = 0;
194
 
194
 
195
    /* compute the BSD sum of the zip package */
195
    /* compute the BSD sum of the zip package */
196
    sprintf(tmpbuf, "%s/%s.zip", repodir, LSMpackage);
196
    sprintf(tmpbuf, "%s/%s.zip", repodir, LSMpackage);
197
    bsum = file2bsum(tmpbuf);
197
    bsum = file2bsum(tmpbuf);
198
 
198
 
199
    printf("Processing %s... BSUM %04X\n", LsmFile, bsum);
199
    printf("Processing %s... BSUM %04X\n", LsmFile, bsum);
200
 
200
 
201
    sprintf(tmpbuf, "appinfo/%s", LsmFile);
201
    sprintf(tmpbuf, "appinfo/%s", LsmFile);
202
    readlsm(tmpbuf, LSMversion, LSMtitle, LSMdescription);
202
    readlsm(tmpbuf, LSMversion, LSMtitle, LSMdescription);
203
 
203
 
204
    if (strlen(LSMpackage) > 8) printf("Warning: %s.zip is not in 8.3 format!\n", LSMpackage);
204
    if (strlen(LSMpackage) > 8) printf("Warning: %s.zip is not in 8.3 format!\n", LSMpackage);
205
    if (LSMtitle[0] == 0) printf("Warning: no LSM title for %s.zip\n", LSMpackage);
205
    if (LSMtitle[0] == 0) printf("Warning: no LSM title for %s.zip\n", LSMpackage);
206
    if (LSMversion[0] == 0) printf("Warning: no LSM version for %s.zip!\n", LSMpackage);
206
    if (LSMversion[0] == 0) printf("Warning: no LSM version for %s.zip!\n", LSMpackage);
207
    if (LSMdescription[0] == 0) printf("Warning: no LSM description for %s.zip!\n", LSMpackage);
207
    if (LSMdescription[0] == 0) printf("Warning: no LSM description for %s.zip!\n", LSMpackage);
208
    fprintf(idx, "%s\t%s\t%s\t%u\n", LSMpackage, LSMversion, LSMdescription, bsum);
208
    fprintf(idx, "%s\t%s\t%s\t%u\n", LSMpackage, LSMversion, LSMdescription, bsum);
209
  }
209
  }
210
  fclose(idx);
210
  fclose(idx);
211
  printf("%d packages found.\n", LsmCount);
211
  printf("%d packages found.\n", LsmCount);
212
}
212
}
213
 
213
 
214
 
214
 
215
 
215
 
216
int main(int argc, char **argv) {
216
int main(int argc, char **argv) {
217
  char *repodir;
217
  char *repodir;
218
  char cmdbuff[1024];
218
  char cmdbuff[256];
219
 
219
 
220
  puts("FDNPKG server repository generator version " pVer);
220
  puts("SvarDOS repository index generator ver " pVer);
221
 
221
 
222
  if (argc != 2) {
222
  if (argc != 2) {
223
    puts("Usage: buildidx repodir");
223
    puts("Usage: buildidx repodir");
224
    return(1);
224
    return(1);
225
  }
225
  }
226
 
226
 
227
  if ((argv[1][0] == '?') || (argv[1][0] == '-') || (argv[1][0]) == 0) {
227
  if ((argv[1][0] == '?') || (argv[1][0] == '-') || (argv[1][0]) == 0) {
228
    puts("Usage: buildidx repodir");
228
    puts("Usage: buildidx repodir");
229
    return(1);
229
    return(1);
230
  }
230
  }
231
 
231
 
232
  repodir = argv[1];
232
  repodir = argv[1];
233
 
233
 
234
  printf("Will build index for %s directory.\n", repodir);
234
  printf("Will build index for %s directory.\n", repodir);
235
  puts("Recreating the appinfo directory...");
235
  puts("Recreating the appinfo directory...");
236
 
236
 
237
  system("rm -rf appinfo");
237
  system("rm -rf appinfo");
238
  system("mkdir appinfo");
238
  system("mkdir appinfo");
239
 
239
 
240
  puts("Populating appinfo with LSM files from archives...");
240
  puts("Populating appinfo with LSM files from archives...");
241
  sprintf(cmdbuff, "unzip -C -j -L -o '%s/*.zip' 'appinfo/*.lsm' -d appinfo", repodir);
241
  snprintf(cmdbuff, sizeof(cmdbuff), "unzip -C -j -L -o '%s/*.zip' 'appinfo/*.lsm' -d appinfo", repodir);
242
  system(cmdbuff);
242
  system(cmdbuff);
243
 
243
 
244
  puts("Generating the index file...");
244
  puts("Generating the index file...");
245
  GenIndexes(repodir);
245
  GenIndexes(repodir);
246
 
246
 
247
  system("rm -rf appinfo");
247
  system("rm -rf appinfo");
248
  return(0);
248
  return(0);
249
}
249
}
250
 
250