Subversion Repositories SvarDOS

Rev

Rev 797 | Rev 801 | Go to most recent revision | Only display areas with differences | Ignore whitespace | Details | Blame | Last modification | View Log | RSS feed

Rev 797 Rev 798
1
<?php /*
1
<?php /*
2
 
2
 
3
  SvarDOS repo index builder
3
  SvarDOS repo index builder
4
  Copyright (C) Mateusz Viste 2012-2022
4
  Copyright (C) Mateusz Viste 2012-2022
5
 
5
 
6
  buildidx computes an index json file for the SvarDOS repository.
6
  buildidx computes an index json file for the SvarDOS repository.
7
  it must be executed pointing to a directory that stores packages (*.svp)
7
  it must be executed pointing to a directory that stores packages (*.svp)
8
  files. buildidx will generate the index file and save it into the package
8
  files. buildidx will generate the index file and save it into the package
9
  repository.
9
  repository.
10
 
10
 
11
  requires php-zip
11
  requires php-zip
12
 
12
 
13
  17 feb 2022: checking for non-8+3 filenames in packages and duplicates + devload no longer part of CORE
13
  17 feb 2022: checking for non-8+3 filenames in packages and duplicates + devload no longer part of CORE
14
  16 feb 2022: added warning about overlong version strings and wild files location
14
  16 feb 2022: added warning about overlong version strings and wild files location
15
  15 feb 2022: index is generated as json, contains all filenames and alt versions
15
  15 feb 2022: index is generated as json, contains all filenames and alt versions
16
  14 feb 2022: packages are expected to have the *.svp extension
16
  14 feb 2022: packages are expected to have the *.svp extension
17
  12 feb 2022: skip source packages from being processed (*.src.zip)
17
  12 feb 2022: skip source packages from being processed (*.src.zip)
18
  20 jan 2022: rewritten the code from ANSI C to PHP for easier maintenance
18
  20 jan 2022: rewritten the code from ANSI C to PHP for easier maintenance
19
  13 feb 2021: 'title' LSM field is no longer looked after
19
  13 feb 2021: 'title' LSM field is no longer looked after
20
  11 feb 2021: lsm headers are no longer checked, so it is compatible with the simpler lsm format used by SvarDOS
20
  11 feb 2021: lsm headers are no longer checked, so it is compatible with the simpler lsm format used by SvarDOS
21
  13 jan 2021: removed the identification line, changed CRC32 to bsum, not creating the listing.txt file and stopped compressing index
21
  13 jan 2021: removed the identification line, changed CRC32 to bsum, not creating the listing.txt file and stopped compressing index
22
  23 apr 2017: uncompressed index is no longer created, added CRC32 of zib (bin only) files, if present
22
  23 apr 2017: uncompressed index is no longer created, added CRC32 of zib (bin only) files, if present
23
  28 aug 2016: listing.txt is always written inside the repo dir (instead of inside current dir)
23
  28 aug 2016: listing.txt is always written inside the repo dir (instead of inside current dir)
24
  27 aug 2016: accepting full paths to repos (starting with /...)
24
  27 aug 2016: accepting full paths to repos (starting with /...)
25
  07 dec 2013: rewritten buildidx in ANSI C89
25
  07 dec 2013: rewritten buildidx in ANSI C89
26
  19 aug 2013: add a compressed version of the index file to repos (index.gz)
26
  19 aug 2013: add a compressed version of the index file to repos (index.gz)
27
  22 jul 2013: creating a listing.txt file with list of packages
27
  22 jul 2013: creating a listing.txt file with list of packages
28
  18 jul 2013: writing the number of packaged into the first line of the lst file
28
  18 jul 2013: writing the number of packaged into the first line of the lst file
29
  11 jul 2013: added a switch to 7za to make it case insensitive when extracting lsm files
29
  11 jul 2013: added a switch to 7za to make it case insensitive when extracting lsm files
30
  10 jul 2013: changed unzip calls to 7za (to handle cases when appinfo is compressed with lzma)
30
  10 jul 2013: changed unzip calls to 7za (to handle cases when appinfo is compressed with lzma)
31
  04 feb 2013: added CRC32 support
31
  04 feb 2013: added CRC32 support
32
  22 sep 2012: forked 1st version from FDUPDATE builder
32
  22 sep 2012: forked 1st version from FDUPDATE builder
33
*/
33
*/
34
 
34
 
35
$PVER = "20220217";
35
$PVER = "20220217";
36
 
36
 
37
 
37
 
38
// computes the BSD sum of a file and returns it
38
// computes the BSD sum of a file and returns it
39
function file2bsum($fname) {
39
function file2bsum($fname) {
40
  $result = 0;
40
  $result = 0;
41
 
41
 
42
  $fd = fopen($fname, 'rb');
42
  $fd = fopen($fname, 'rb');
43
  if ($fd === false) return(0);
43
  if ($fd === false) return(0);
44
 
44
 
45
  while (!feof($fd)) {
45
  while (!feof($fd)) {
46
 
46
 
47
    $buff = fread($fd, 1024 * 1024);
47
    $buff = fread($fd, 1024 * 1024);
48
 
48
 
49
    $slen = strlen($buff);
49
    $slen = strlen($buff);
50
    for ($i = 0; $i < $slen; $i++) {
50
    for ($i = 0; $i < $slen; $i++) {
51
      // rotr
51
      // rotr
52
      $result = ($result >> 1) | ($result << 15);
52
      $result = ($result >> 1) | ($result << 15);
53
      // add and truncate to 16 bits
53
      // add and truncate to 16 bits
54
      $result += ord($buff[$i]);
54
      $result += ord($buff[$i]);
55
      $result &= 0xffff;
55
      $result &= 0xffff;
56
    }
56
    }
57
  }
57
  }
58
 
58
 
59
  fclose($fd);
59
  fclose($fd);
60
  return($result);
60
  return($result);
61
}
61
}
62
 
62
 
63
 
63
 
64
// reads file fil from zip archive z and returns its content, or false on error
64
// reads file fil from zip archive z and returns its content, or false on error
65
function read_file_from_zip($z, $fil) {
65
function read_file_from_zip($z, $fil) {
66
  $zip = new ZipArchive;
66
  $zip = new ZipArchive;
67
  if ($zip->open($z, ZipArchive::RDONLY) !== true) {
67
  if ($zip->open($z, ZipArchive::RDONLY) !== true) {
68
    echo "ERROR: failed to open zip file '{$z}'\n";
68
    echo "ERROR: failed to open zip file '{$z}'\n";
69
    return(false);
69
    return(false);
70
  }
70
  }
71
 
71
 
72
  // load the appinfo/pkgname.lsm file
72
  // load the appinfo/pkgname.lsm file
73
  $res = $zip->getFromName($fil, 8192, ZipArchive::FL_NOCASE);
73
  $res = $zip->getFromName($fil, 8192, ZipArchive::FL_NOCASE);
74
 
74
 
75
  $zip->close();
75
  $zip->close();
76
  return($res);
76
  return($res);
77
}
77
}
78
 
78
 
79
 
79
 
80
function read_list_of_files_in_zip($z) {
80
function read_list_of_files_in_zip($z) {
81
  $zip = new ZipArchive;
81
  $zip = new ZipArchive;
82
  if ($zip->open($z, ZipArchive::RDONLY) !== true) {
82
  if ($zip->open($z, ZipArchive::RDONLY) !== true) {
83
    echo "ERROR: failed to open zip file '{$z}'\n";
83
    echo "ERROR: failed to open zip file '{$z}'\n";
84
    return(false);
84
    return(false);
85
  }
85
  }
86
 
86
 
87
  $res = array();
87
  $res = array();
88
  for ($i = 0; $i < $zip->numFiles; $i++) $res[] = $zip->getNameIndex($i);
88
  for ($i = 0; $i < $zip->numFiles; $i++) $res[] = $zip->getNameIndex($i);
89
 
89
 
90
  $zip->close();
90
  $zip->close();
91
  return($res);
91
  return($res);
92
}
92
}
93
 
93
 
94
 
94
 
95
// reads a LSM string and returns it in the form of an array
95
// reads a LSM string and returns it in the form of an array
96
function parse_lsm($s) {
96
function parse_lsm($s) {
97
  $res = array();
97
  $res = array();
98
  for ($l = strtok($s, "\n"); $l !== false; $l = strtok("\n")) {
98
  for ($l = strtok($s, "\n"); $l !== false; $l = strtok("\n")) {
99
    // the line is "token: value", let's find the colon
99
    // the line is "token: value", let's find the colon
100
    $colpos = strpos($l, ':');
100
    $colpos = strpos($l, ':');
101
    if (($colpos === false) || ($colpos === 0)) continue;
101
    if (($colpos === false) || ($colpos === 0)) continue;
102
    $tok = strtolower(trim(substr($l, 0, $colpos)));
102
    $tok = strtolower(trim(substr($l, 0, $colpos)));
103
    $val = trim(substr($l, $colpos + 1));
103
    $val = trim(substr($l, $colpos + 1));
104
    $res[$tok] = $val;
104
    $res[$tok] = $val;
105
  }
105
  }
106
  return($res);
106
  return($res);
107
}
107
}
108
 
108
 
109
 
109
 
110
// on PHP 8+ there is str_starts_with(), but not on PHP 7 so I use this
110
// on PHP 8+ there is str_starts_with(), but not on PHP 7 so I use this
111
function str_head_is($haystack, $needle) {
111
function str_head_is($haystack, $needle) {
112
  return strpos($haystack, $needle) === 0;
112
  return strpos($haystack, $needle) === 0;
113
}
113
}
114
 
114
 
115
 
115
 
116
// returns an array that contains CORE packages (populated from the core subdirectory in pkgdir)
116
// returns an array that contains CORE packages (populated from the core subdirectory in pkgdir)
117
function load_core_list($repodir) {
117
function load_core_list($repodir) {
118
  $res = array();
118
  $res = array();
119
 
119
 
120
  foreach (scandir($repodir . '/core/') as $f) {
120
  foreach (scandir($repodir . '/core/') as $f) {
121
    if (!preg_match('/\.svp$/', $f)) continue;
121
    if (!preg_match('/\.svp$/', $f)) continue;
122
    $res[] = explode('.', $f)[0];
122
    $res[] = explode('.', $f)[0];
123
  }
123
  }
124
  return($res);
124
  return($res);
125
}
125
}
126
 
126
 
127
 
127
 
128
// ***************** MAIN ROUTINE *********************************************
128
// ***************** MAIN ROUTINE *********************************************
129
 
129
 
130
//echo "SvarDOS repository index generator ver {$PVER}\n";
130
//echo "SvarDOS repository index generator ver {$PVER}\n";
131
 
131
 
132
if (($_SERVER['argc'] != 2) || ($_SERVER['argv'][1][0] == '-')) {
132
if (($_SERVER['argc'] != 2) || ($_SERVER['argv'][1][0] == '-')) {
133
  echo "usage: php buildidx.php repodir\n";
133
  echo "usage: php buildidx.php repodir\n";
134
  exit(1);
134
  exit(1);
135
}
135
}
136
 
136
 
137
$repodir = $_SERVER['argv'][1];
137
$repodir = $_SERVER['argv'][1];
138
 
138
 
139
$pkgfiles = scandir($repodir);
139
$pkgfiles = scandir($repodir);
140
$pkgcount = 0;
140
$pkgcount = 0;
141
 
141
 
142
 
142
 
143
// load the list of CORE and MSDOS_COMPAT packages
143
// load the list of CORE and MSDOS_COMPAT packages
144
 
144
 
145
$core_packages_list = load_core_list($repodir);
145
$core_packages_list = load_core_list($repodir);
146
$msdos_compat_list = explode(' ', 'append assign attrib chkdsk choice command cpidos debug defrag deltree diskcopy display edit edlin fc fdapm fdisk find format help himemx kernel keyb label localcfg mem mode more move nlsfunc print replace share shsucdx sort swsubst tree undelete unformat xcopy');
146
$msdos_compat_list = explode(' ', 'append assign attrib chkdsk choice command cpidos debug defrag deltree diskcopy display edit edlin fc fdapm fdisk find format help himemx kernel keyb label localcfg mem mode more move nlsfunc print replace share shsucdx sort swsubst tree undelete unformat xcopy');
147
 
147
 
148
// do a list of all svp packages with their available versions and descriptions
148
// do a list of all svp packages with their available versions and descriptions
149
 
149
 
150
$pkgdb = array();
150
$pkgdb = array();
151
foreach ($pkgfiles as $fname) {
151
foreach ($pkgfiles as $fname) {
152
  if (!preg_match('/.svp$/i', $fname)) continue; // skip non-svp files
152
  if (!preg_match('/.svp$/i', $fname)) continue; // skip non-svp files
153
 
153
 
154
  $path_parts = pathinfo($fname);
154
  $path_parts = pathinfo($fname);
155
  $pkgnam = explode('-', $path_parts['filename'])[0];
155
  $pkgnam = explode('-', $path_parts['filename'])[0];
156
  $pkgfullpath = realpath($repodir . '/' . $fname);
156
  $pkgfullpath = realpath($repodir . '/' . $fname);
157
 
157
 
158
  $lsm = read_file_from_zip($pkgfullpath, "appinfo/{$pkgnam}.lsm");
158
  $lsm = read_file_from_zip($pkgfullpath, "appinfo/{$pkgnam}.lsm");
159
  if ($lsm == false) {
159
  if ($lsm == false) {
160
    echo "ERROR: pkg {$fname} does not contain an LSM file at the expected location\n";
160
    echo "ERROR: pkg {$fname} does not contain an LSM file at the expected location\n";
161
    continue;
161
    continue;
162
  }
162
  }
163
  $lsmarray = parse_lsm($lsm);
163
  $lsmarray = parse_lsm($lsm);
164
  if (empty($lsmarray['version'])) {
164
  if (empty($lsmarray['version'])) {
165
    echo "ERROR: lsm file in {$fname} does not contain a version\n";
165
    echo "ERROR: lsm file in {$fname} does not contain a version\n";
166
    continue;
166
    continue;
167
  }
167
  }
168
  if (strlen($lsmarray['version']) > 16) {
168
  if (strlen($lsmarray['version']) > 16) {
169
    echo "ERROR: version string in lsm file of {$fname} is too long (16 chars max)\n";
169
    echo "ERROR: version string in lsm file of {$fname} is too long (16 chars max)\n";
170
    continue;
170
    continue;
171
  }
171
  }
172
  if (empty($lsmarray['description'])) {
172
  if (empty($lsmarray['description'])) {
173
    echo "ERROR: lsm file in {$fname} does not contain a description\n";
173
    echo "ERROR: lsm file in {$fname} does not contain a description\n";
174
    continue;
174
    continue;
175
  }
175
  }
176
 
176
 
177
  // validate the files present in the archive
177
  // validate the files present in the archive
178
  $listoffiles = read_list_of_files_in_zip($pkgfullpath);
178
  $listoffiles = read_list_of_files_in_zip($pkgfullpath);
179
  $pkgdir = $pkgnam;
179
  $pkgdir = $pkgnam;
180
 
180
 
181
  // special rule for "parent and children" packages
181
  // special rule for "parent and children" packages
182
  if (str_head_is($pkgnam, 'djgpp_')) $pkgdir = 'djgpp'; // djgpp_* packages put their files in djgpp
182
  if (str_head_is($pkgnam, 'djgpp_')) $pkgdir = 'djgpp'; // djgpp_* packages put their files in djgpp
183
  if ($pkgnam == 'fbc_help') $pkgdir = 'fbc'; // FreeBASIC help goes to the FreeBASIC dir
183
  if ($pkgnam == 'fbc_help') $pkgdir = 'fbc'; // FreeBASIC help goes to the FreeBASIC dir
184
 
184
 
185
  // array used to detect duplicated entries after lower-case conversion
185
  // array used to detect duplicated entries after lower-case conversion
186
  $duparr = array();
186
  $duparr = array();
187
 
187
 
188
  foreach ($listoffiles as $f) {
188
  foreach ($listoffiles as $f) {
189
    $f = strtolower($f);
189
    $f = strtolower($f);
190
    $path_array = explode('/', $f);
190
    $path_array = explode('/', $f);
191
    // emit a warning when non-8+3 filenames are spotted and find duplicates
191
    // emit a warning when non-8+3 filenames are spotted and find duplicates
192
    foreach ($path_array as $item) {
192
    foreach ($path_array as $item) {
193
      if (empty($item)) continue; // skip empty items at end of paths (eg. appinfo/)
193
      if (empty($item)) continue; // skip empty items at end of paths (eg. appinfo/)
194
      if (!preg_match("/[a-z0-9!#$%&'()@^_`{}~-]{1,8}(\.[a-z0-9!#$%&'()@^_`{}~-]{1,3}){0,1}/", $item)) {
194
      if (!preg_match("/[a-z0-9!#$%&'()@^_`{}~-]{1,8}(\.[a-z0-9!#$%&'()@^_`{}~-]{1,3}){0,1}/", $item)) {
195
        echo "WARNING: {$fname} contains a non-8+3 path (or weird char): {$item} (in $f)\n";
195
        echo "WARNING: {$fname} contains a non-8+3 path (or weird char): {$item} (in $f)\n";
196
      }
196
      }
197
    }
197
    }
198
    // look for dups
198
    // look for dups
199
    if (array_search($f, $duparr) !== false) {
199
    if (array_search($f, $duparr) !== false) {
200
      echo "WARNING: {$fname} contains a duplicated entry: '{$f}'\n";
200
      echo "WARNING: {$fname} contains a duplicated entry: '{$f}'\n";
201
    } else {
201
    } else {
202
      $duparr[] = $f;
202
      $duparr[] = $f;
203
    }
203
    }
204
    // LSM file is ok
204
    // LSM file is ok
205
    if ($f === "appinfo/{$pkgnam}.lsm") continue;
205
    if ($f === "appinfo/{$pkgnam}.lsm") continue;
206
    if ($f === "appinfo/") continue;
206
    if ($f === "appinfo/") continue;
207
    // CORE and MSDOS_COMPAT packages are premium citizens and can do a little more
207
    // CORE and MSDOS_COMPAT packages are premium citizens and can do a little more
208
    if ((array_search($pkgnam, $core_packages_list) !== false)
208
    if ((array_search($pkgnam, $core_packages_list) !== false)
209
       || (array_search($pkgnam, $msdos_compat_list) !== false)) {
209
       || (array_search($pkgnam, $msdos_compat_list) !== false)) {
210
      if (str_head_is($f, 'bin/')) continue;
210
      if (str_head_is($f, 'bin/')) continue;
211
      if (str_head_is($f, 'cpi/')) continue;
211
      if (str_head_is($f, 'cpi/')) continue;
212
      if (str_head_is($f, "doc/{$pkgdir}/")) continue;
212
      if (str_head_is($f, "doc/{$pkgdir}/")) continue;
213
      if ($f === 'doc/') continue;
213
      if ($f === 'doc/') continue;
214
      if (str_head_is($f, "nls/{$pkgdir}.")) continue;
214
      if (str_head_is($f, "nls/{$pkgdir}.")) continue;
215
      if ($f === 'nls/') continue;
215
      if ($f === 'nls/') continue;
216
    }
216
    }
-
 
217
    // the help package is allowed to put files in... help
-
 
218
    if (($pkgnam == 'help') && (str_head_is($f, 'help/'))) continue;
217
    // well-known "category" dirs are okay
219
    // well-known "category" dirs are okay
218
    if (str_head_is($f, "progs/{$pkgdir}/")) continue;
220
    if (str_head_is($f, "progs/{$pkgdir}/")) continue;
219
    if ($f === 'progs/') continue;
221
    if ($f === 'progs/') continue;
220
    if (str_head_is($f, "devel/{$pkgdir}/")) continue;
222
    if (str_head_is($f, "devel/{$pkgdir}/")) continue;
221
    if ($f === 'devel/') continue;
223
    if ($f === 'devel/') continue;
222
    if (str_head_is($f, "games/{$pkgdir}/")) continue;
224
    if (str_head_is($f, "games/{$pkgdir}/")) continue;
223
    if ($f === 'games/') continue;
225
    if ($f === 'games/') continue;
224
    if (str_head_is($f, "drivers/{$pkgdir}/")) continue;
226
    if (str_head_is($f, "drivers/{$pkgdir}/")) continue;
225
    if ($f === 'drivers/') continue;
227
    if ($f === 'drivers/') continue;
226
    echo "WARNING: {$fname} contains a file in an illegal location: {$f}\n";
228
    echo "WARNING: {$fname} contains a file in an illegal location: {$f}\n";
227
  }
229
  }
228
 
230
 
229
  $meta['fname'] = $fname;
231
  $meta['fname'] = $fname;
230
  $meta['desc'] = $lsmarray['description'];
232
  $meta['desc'] = $lsmarray['description'];
231
 
233
 
232
  $pkgdb[$pkgnam][$lsmarray['version']] = $meta;
234
  $pkgdb[$pkgnam][$lsmarray['version']] = $meta;
233
}
235
}
234
 
236
 
235
$db = array();
237
$db = array();
236
 
238
 
237
// iterate over each svp package
239
// iterate over each svp package
238
foreach ($pkgdb as $pkg => $versions) {
240
foreach ($pkgdb as $pkg => $versions) {
239
 
241
 
240
  // sort filenames by version, highest first
242
  // sort filenames by version, highest first
241
  uksort($versions, "version_compare");
243
  uksort($versions, "version_compare");
242
  $versions = array_reverse($versions, true);
244
  $versions = array_reverse($versions, true);
243
 
245
 
244
  foreach ($versions as $ver => $meta) {
246
  foreach ($versions as $ver => $meta) {
245
    $fname = $meta['fname'];
247
    $fname = $meta['fname'];
246
    $desc = $meta['desc'];
248
    $desc = $meta['desc'];
247
 
249
 
248
    $bsum = file2bsum(realpath($repodir . '/' . $fname));
250
    $bsum = file2bsum(realpath($repodir . '/' . $fname));
249
 
251
 
250
    $meta2['ver'] = strval($ver);
252
    $meta2['ver'] = strval($ver);
251
    $meta2['bsum'] = $bsum;
253
    $meta2['bsum'] = $bsum;
252
 
254
 
253
    if (empty($db[$pkg]['desc'])) $db[$pkg]['desc'] = $desc;
255
    if (empty($db[$pkg]['desc'])) $db[$pkg]['desc'] = $desc;
254
    $db[$pkg]['versions'][$fname] = $meta2;
256
    $db[$pkg]['versions'][$fname] = $meta2;
255
  }
257
  }
256
 
258
 
257
  $pkgcount++;
259
  $pkgcount++;
258
 
260
 
259
}
261
}
260
 
262
 
261
if ($pkgcount < 100) echo "WARNING: an unexpectedly low number of packages has been found in the repo ({$pkgcount})\n";
263
if ($pkgcount < 100) echo "WARNING: an unexpectedly low number of packages has been found in the repo ({$pkgcount})\n";
262
 
264
 
263
file_put_contents($repodir . '/_index.json', json_encode($db));
265
file_put_contents($repodir . '/_index.json', json_encode($db));
264
 
266
 
265
exit(0);
267
exit(0);
266
 
268
 
267
?>
269
?>
268
 
270