Subversion Repositories SvarDOS

Rev

Rev 909 | Rev 914 | Go to most recent revision | Details | Compare with Previous | Last modification | View Log | RSS feed

Rev Author Line No. Line
562 mateuszvis 1
<?php /*
2
 
3
  SvarDOS repo index builder
4
  Copyright (C) Mateusz Viste 2012-2022
5
 
734 bttr 6
  buildidx computes an index json file for the SvarDOS repository.
673 mateusz.vi 7
  it must be executed pointing to a directory that stores packages (*.svp)
562 mateuszvis 8
  files. buildidx will generate the index file and save it into the package
9
  repository.
10
 
11
  requires php-zip
12
 
912 mateusz.vi 13
  21 feb 2022: buildidx collects categories looking at the dir layout of each package + improved version string parsing (replaced version_compare call by dos_version_compare)
775 mateusz.vi 14
  17 feb 2022: checking for non-8+3 filenames in packages and duplicates + devload no longer part of CORE
736 mateusz.vi 15
  16 feb 2022: added warning about overlong version strings and wild files location
719 mateusz.vi 16
  15 feb 2022: index is generated as json, contains all filenames and alt versions
673 mateusz.vi 17
  14 feb 2022: packages are expected to have the *.svp extension
650 mateusz.vi 18
  12 feb 2022: skip source packages from being processed (*.src.zip)
562 mateuszvis 19
  20 jan 2022: rewritten the code from ANSI C to PHP for easier maintenance
20
  13 feb 2021: 'title' LSM field is no longer looked after
21
  11 feb 2021: lsm headers are no longer checked, so it is compatible with the simpler lsm format used by SvarDOS
22
  13 jan 2021: removed the identification line, changed CRC32 to bsum, not creating the listing.txt file and stopped compressing index
23
  23 apr 2017: uncompressed index is no longer created, added CRC32 of zib (bin only) files, if present
24
  28 aug 2016: listing.txt is always written inside the repo dir (instead of inside current dir)
25
  27 aug 2016: accepting full paths to repos (starting with /...)
26
  07 dec 2013: rewritten buildidx in ANSI C89
27
  19 aug 2013: add a compressed version of the index file to repos (index.gz)
28
  22 jul 2013: creating a listing.txt file with list of packages
29
  18 jul 2013: writing the number of packaged into the first line of the lst file
30
  11 jul 2013: added a switch to 7za to make it case insensitive when extracting lsm files
31
  10 jul 2013: changed unzip calls to 7za (to handle cases when appinfo is compressed with lzma)
32
  04 feb 2013: added CRC32 support
33
  22 sep 2012: forked 1st version from FDUPDATE builder
34
*/
35
 
909 mateusz.vi 36
$PVER = "20220221";
562 mateuszvis 37
 
38
 
39
// computes the BSD sum of a file and returns it
40
function file2bsum($fname) {
41
  $result = 0;
42
 
43
  $fd = fopen($fname, 'rb');
44
  if ($fd === false) return(0);
45
 
46
  while (!feof($fd)) {
47
 
48
    $buff = fread($fd, 1024 * 1024);
49
 
563 mateuszvis 50
    $slen = strlen($buff);
51
    for ($i = 0; $i < $slen; $i++) {
562 mateuszvis 52
      // rotr
53
      $result = ($result >> 1) | ($result << 15);
54
      // add and truncate to 16 bits
563 mateuszvis 55
      $result += ord($buff[$i]);
562 mateuszvis 56
      $result &= 0xffff;
57
    }
58
  }
59
 
60
  fclose($fd);
61
  return($result);
62
}
63
 
64
 
912 mateusz.vi 65
// translates a version string into a array of integer values.
66
// Accepted formats follow:
67
//    300.12.1
68
//    1
69
//    12.2.34.2-4.5
70
//    1.2c
71
//    1.01 beta+3
72
//    2013-12-31
73
//    20220222 alpha
74
function vertoarr($verstr) {
75
  $subver = array(0,0,0,0);
76
 
77
  // switch string to lcase for easier processing and trim any leading or trailing white spaces
78
  $verstr = strtolower(trim($verstr));
79
 
80
  // replace all '-' and '/' characters to '.' (uniformization of sub-version parts delimiters)
81
  $verstr = strtr($verstr, '-/', '..');
82
 
83
  // is there a subversion value? (for example "+4" in "1.05+4")
84
  $i = strrpos($verstr, '+', 1);
85
  if ($i !== false) {
86
    // validate the svar-version is a proper integer
87
    $svarver = substr($verstr, $i + 1);
88
    if (! preg_match('/[1-9][0-9]*/', $svarver)) {
89
      return(false);
90
    }
91
    $subver[3] = intval($svarver); // set the +rev as a very minor item
92
    $verstr = substr($verstr, 0, $i);
93
  }
94
 
95
  // is the version ending with ' alpha', 'beta'?
96
  if (preg_match('/ (alpha|beta)$/', $verstr)) {
97
    $i = strrpos($verstr, ' ');
98
    $greek = substr($verstr, $i + 1);
99
    $verstr = trim(substr($verstr, 0, $i));
100
    if ($greek == 'alpha') {
101
      $subver[2] = 1;
102
    } else if ($greek == 'beta') {
103
      $subver[2] = 2;
104
    } else {
105
      return(false);
106
    }
107
  }
108
 
109
  // does the version string have a single-letter subversion? (1.0c)
110
  if (preg_match('/[a-z]$/', $verstr)) {
111
    $subver[1] = ord(substr($verstr, -1));
112
    $verstr = substr_replace($verstr, '', -1); // remove last character from string
113
  }
114
 
115
  // validate the format is supported, should be something no more complex than 1.05.3.33
116
  if (! preg_match('/[0-9][0-9.]{0,20}/', $verstr)) {
117
    return(false);
118
  }
119
 
120
  // NOTE: a zero right after a separator and trailed with a digit (as in 1.01)
121
  //       has a special meaning
122
  $exploded = explode('.', $verstr);
123
  if (count($exploded) > 16) {
124
    return(false);
125
  }
126
  $exploded[16] = $subver[0]; // unused yet
127
  $exploded[17] = $subver[1]; // a-z (1.0c)
128
  $exploded[18] = $subver[2]; // alpha/beta
129
  $exploded[19] = $subver[3]; // svar-ver (1.0+5)
130
  for ($i = 0; $i < 20; $i++) if (empty($exploded[$i])) $exploded[$i] = '0';
131
 
132
  ksort($exploded);
133
 
134
  return($exploded);
135
}
136
 
137
 
138
function dos_version_compare($v1, $v2) {
139
  $v1arr = vertoarr($v1);
140
  $v2arr = vertoarr($v2);
141
  for ($i = 0; $i < count($v1arr); $i++) {
142
    $r = strcmp($v1arr[$i], $v2arr[$i]);
143
    if ($r != 0) return($r);
144
  }
145
  return(0);
146
}
147
 
148
 
562 mateuszvis 149
// reads file fil from zip archive z and returns its content, or false on error
150
function read_file_from_zip($z, $fil) {
151
  $zip = new ZipArchive;
152
  if ($zip->open($z, ZipArchive::RDONLY) !== true) {
153
    echo "ERROR: failed to open zip file '{$z}'\n";
154
    return(false);
155
  }
156
 
157
  // load the appinfo/pkgname.lsm file
158
  $res = $zip->getFromName($fil, 8192, ZipArchive::FL_NOCASE);
159
 
160
  $zip->close();
161
  return($res);
162
}
163
 
164
 
731 mateusz.vi 165
function read_list_of_files_in_zip($z) {
166
  $zip = new ZipArchive;
167
  if ($zip->open($z, ZipArchive::RDONLY) !== true) {
168
    echo "ERROR: failed to open zip file '{$z}'\n";
169
    return(false);
170
  }
171
 
172
  $res = array();
173
  for ($i = 0; $i < $zip->numFiles; $i++) $res[] = $zip->getNameIndex($i);
174
 
175
  $zip->close();
176
  return($res);
177
}
178
 
179
 
562 mateuszvis 180
// reads a LSM string and returns it in the form of an array
181
function parse_lsm($s) {
182
  $res = array();
183
  for ($l = strtok($s, "\n"); $l !== false; $l = strtok("\n")) {
184
    // the line is "token: value", let's find the colon
185
    $colpos = strpos($l, ':');
186
    if (($colpos === false) || ($colpos === 0)) continue;
187
    $tok = strtolower(trim(substr($l, 0, $colpos)));
188
    $val = trim(substr($l, $colpos + 1));
189
    $res[$tok] = $val;
190
  }
191
  return($res);
192
}
193
 
194
 
731 mateusz.vi 195
// on PHP 8+ there is str_starts_with(), but not on PHP 7 so I use this
196
function str_head_is($haystack, $needle) {
197
  return strpos($haystack, $needle) === 0;
198
}
199
 
200
 
791 mateusz.vi 201
// returns an array that contains CORE packages (populated from the core subdirectory in pkgdir)
202
function load_core_list($repodir) {
203
  $res = array();
204
 
205
  foreach (scandir($repodir . '/core/') as $f) {
206
    if (!preg_match('/\.svp$/', $f)) continue;
207
    $res[] = explode('.', $f)[0];
208
  }
209
  return($res);
210
}
211
 
212
 
562 mateuszvis 213
// ***************** MAIN ROUTINE *********************************************
214
 
719 mateusz.vi 215
//echo "SvarDOS repository index generator ver {$PVER}\n";
562 mateuszvis 216
 
217
if (($_SERVER['argc'] != 2) || ($_SERVER['argv'][1][0] == '-')) {
218
  echo "usage: php buildidx.php repodir\n";
219
  exit(1);
220
}
221
 
222
$repodir = $_SERVER['argv'][1];
223
 
224
$pkgfiles = scandir($repodir);
225
$pkgcount = 0;
226
 
738 mateusz.vi 227
 
795 mateusz.vi 228
// load the list of CORE and MSDOS_COMPAT packages
738 mateusz.vi 229
 
791 mateusz.vi 230
$core_packages_list = load_core_list($repodir);
804 bttr 231
$msdos_compat_list = explode(' ', 'append assign attrib chkdsk choice command comp cpidos debug defrag deltree diskcomp diskcopy display edit edlin exe2bin fc fdapm fdisk find format help himemx kernel keyb label localcfg mem mirror mode more move nlsfunc print replace share shsucdx sort swsubst tree undelete unformat xcopy');
738 mateusz.vi 232
 
719 mateusz.vi 233
// do a list of all svp packages with their available versions and descriptions
562 mateuszvis 234
 
719 mateusz.vi 235
$pkgdb = array();
236
foreach ($pkgfiles as $fname) {
801 mateusz.vi 237
  if (!preg_match('/\.svp$/i', $fname)) continue; // skip non-svp files
562 mateuszvis 238
 
801 mateusz.vi 239
  if (!preg_match('/^[a-zA-Z0-9+. _-]*\.svp$/', $fname)) {
240
    echo "ERROR: {$fname} has a very weird name\n";
241
    continue;
242
  }
243
 
719 mateusz.vi 244
  $path_parts = pathinfo($fname);
245
  $pkgnam = explode('-', $path_parts['filename'])[0];
246
  $pkgfullpath = realpath($repodir . '/' . $fname);
562 mateuszvis 247
 
719 mateusz.vi 248
  $lsm = read_file_from_zip($pkgfullpath, "appinfo/{$pkgnam}.lsm");
562 mateuszvis 249
  if ($lsm == false) {
802 mateusz.vi 250
    echo "ERROR: {$fname} does not contain an LSM file at the expected location\n";
719 mateusz.vi 251
    continue;
562 mateuszvis 252
  }
253
  $lsmarray = parse_lsm($lsm);
254
  if (empty($lsmarray['version'])) {
719 mateusz.vi 255
    echo "ERROR: lsm file in {$fname} does not contain a version\n";
256
    continue;
562 mateuszvis 257
  }
730 mateusz.vi 258
  if (strlen($lsmarray['version']) > 16) {
737 mateusz.vi 259
    echo "ERROR: version string in lsm file of {$fname} is too long (16 chars max)\n";
730 mateusz.vi 260
    continue;
261
  }
562 mateuszvis 262
  if (empty($lsmarray['description'])) {
719 mateusz.vi 263
    echo "ERROR: lsm file in {$fname} does not contain a description\n";
264
    continue;
562 mateuszvis 265
  }
266
 
731 mateusz.vi 267
  // validate the files present in the archive
268
  $listoffiles = read_list_of_files_in_zip($pkgfullpath);
739 mateusz.vi 269
  $pkgdir = $pkgnam;
270
 
768 mateusz.vi 271
  // special rule for "parent and children" packages
272
  if (str_head_is($pkgnam, 'djgpp_')) $pkgdir = 'djgpp'; // djgpp_* packages put their files in djgpp
754 mateusz.vi 273
  if ($pkgnam == 'fbc_help') $pkgdir = 'fbc'; // FreeBASIC help goes to the FreeBASIC dir
802 mateusz.vi 274
  if ($pkgnam == 'clamdb') $pkgdir = 'clamav'; // data patterns for clamav
739 mateusz.vi 275
 
768 mateusz.vi 276
  // array used to detect duplicated entries after lower-case conversion
277
  $duparr = array();
278
 
909 mateusz.vi 279
  // will hold the list of categories that this package belongs to
280
  $catlist = array();
281
 
731 mateusz.vi 282
  foreach ($listoffiles as $f) {
283
    $f = strtolower($f);
768 mateusz.vi 284
    $path_array = explode('/', $f);
285
    // emit a warning when non-8+3 filenames are spotted and find duplicates
286
    foreach ($path_array as $item) {
287
      if (empty($item)) continue; // skip empty items at end of paths (eg. appinfo/)
288
      if (!preg_match("/[a-z0-9!#$%&'()@^_`{}~-]{1,8}(\.[a-z0-9!#$%&'()@^_`{}~-]{1,3}){0,1}/", $item)) {
289
        echo "WARNING: {$fname} contains a non-8+3 path (or weird char): {$item} (in $f)\n";
290
      }
291
    }
292
    // look for dups
293
    if (array_search($f, $duparr) !== false) {
294
      echo "WARNING: {$fname} contains a duplicated entry: '{$f}'\n";
295
    } else {
296
      $duparr[] = $f;
297
    }
731 mateusz.vi 298
    // LSM file is ok
299
    if ($f === "appinfo/{$pkgnam}.lsm") continue;
300
    if ($f === "appinfo/") continue;
795 mateusz.vi 301
    // CORE and MSDOS_COMPAT packages are premium citizens and can do a little more
909 mateusz.vi 302
    $core_or_msdoscompat = 0;
303
    if (array_search($pkgnam, $core_packages_list) !== false) {
304
      $catlist[] = 'core';
305
      $core_or_msdoscompat = 1;
306
    }
307
    if (array_search($pkgnam, $msdos_compat_list) !== false) {
308
      $catlist[] = 'msdos_compat';
309
      $core_or_msdoscompat = 1;
310
    }
311
    if ($core_or_msdoscompat == 1) {
736 mateusz.vi 312
      if (str_head_is($f, 'bin/')) continue;
779 mateusz.vi 313
      if (str_head_is($f, 'cpi/')) continue;
749 mateusz.vi 314
      if (str_head_is($f, "doc/{$pkgdir}/")) continue;
315
      if ($f === 'doc/') continue;
316
      if (str_head_is($f, "nls/{$pkgdir}.")) continue;
317
      if ($f === 'nls/') continue;
736 mateusz.vi 318
    }
798 mateusz.vi 319
    // the help package is allowed to put files in... help
320
    if (($pkgnam == 'help') && (str_head_is($f, 'help/'))) continue;
909 mateusz.vi 321
    // must be category-prefixed file, add it to the list of categories for this package
322
    $catlist[] = explode('/', $f)[0];
749 mateusz.vi 323
    // well-known "category" dirs are okay
739 mateusz.vi 324
    if (str_head_is($f, "progs/{$pkgdir}/")) continue;
731 mateusz.vi 325
    if ($f === 'progs/') continue;
739 mateusz.vi 326
    if (str_head_is($f, "devel/{$pkgdir}/")) continue;
731 mateusz.vi 327
    if ($f === 'devel/') continue;
739 mateusz.vi 328
    if (str_head_is($f, "games/{$pkgdir}/")) continue;
731 mateusz.vi 329
    if ($f === 'games/') continue;
739 mateusz.vi 330
    if (str_head_is($f, "drivers/{$pkgdir}/")) continue;
731 mateusz.vi 331
    if ($f === 'drivers/') continue;
768 mateusz.vi 332
    echo "WARNING: {$fname} contains a file in an illegal location: {$f}\n";
731 mateusz.vi 333
  }
334
 
912 mateusz.vi 335
  // do I understand the version string?
336
  if (vertoarr($lsmarray['version']) === false) echo "WARNING: {$fname} parsing of version string failed ('{$lsmarray['version']}')\n";
337
 
719 mateusz.vi 338
  $meta['fname'] = $fname;
339
  $meta['desc'] = $lsmarray['description'];
909 mateusz.vi 340
  $meta['cats'] = array_unique($catlist);
719 mateusz.vi 341
 
342
  $pkgdb[$pkgnam][$lsmarray['version']] = $meta;
343
}
344
 
801 mateusz.vi 345
 
719 mateusz.vi 346
$db = array();
909 mateusz.vi 347
$cats = array();
719 mateusz.vi 348
 
909 mateusz.vi 349
// ******** compute the version-sorted list of packages with a single *********
350
// ******** description and category list for each package ********************
351
 
719 mateusz.vi 352
// iterate over each svp package
353
foreach ($pkgdb as $pkg => $versions) {
354
 
355
  // sort filenames by version, highest first
912 mateusz.vi 356
  uksort($versions, "dos_version_compare");
719 mateusz.vi 357
  $versions = array_reverse($versions, true);
358
 
359
  foreach ($versions as $ver => $meta) {
360
    $fname = $meta['fname'];
361
    $desc = $meta['desc'];
362
 
363
    $bsum = file2bsum(realpath($repodir . '/' . $fname));
364
 
365
    $meta2['ver'] = strval($ver);
366
    $meta2['bsum'] = $bsum;
367
 
368
    if (empty($db[$pkg]['desc'])) $db[$pkg]['desc'] = $desc;
909 mateusz.vi 369
    if (empty($db[$pkg]['cats'])) {
370
      $db[$pkg]['cats'] = $meta['cats'];
371
      $cats = array_unique(array_merge($cats, $meta['cats']));
372
    }
719 mateusz.vi 373
    $db[$pkg]['versions'][$fname] = $meta2;
374
  }
375
 
562 mateuszvis 376
  $pkgcount++;
377
 
378
}
379
 
719 mateusz.vi 380
if ($pkgcount < 100) echo "WARNING: an unexpectedly low number of packages has been found in the repo ({$pkgcount})\n";
562 mateuszvis 381
 
801 mateusz.vi 382
$json_blob = json_encode($db);
383
if ($json_blob === false) {
384
  echo "ERROR: JSON convertion failed! -> ";
385
  switch (json_last_error()) {
386
    case JSON_ERROR_DEPTH:
387
      echo 'maximum stack depth exceeded';
388
      break;
389
    case JSON_ERROR_STATE_MISMATCH:
390
      echo 'underflow of the modes mismatch';
391
      break;
392
    case JSON_ERROR_CTRL_CHAR:
393
      echo 'unexpected control character found';
394
      break;
395
    case JSON_ERROR_UTF8:
396
      echo 'malformed utf-8 characters';
397
      break;
398
    default:
399
      echo "unknown error";
400
      break;
401
  }
402
  echo "\n";
403
}
404
 
909 mateusz.vi 405
file_put_contents($repodir . '/_index.json', $json_blob);
562 mateuszvis 406
 
909 mateusz.vi 407
$cats_json = json_encode($cats);
408
file_put_contents($repodir . '/_cats.json', $cats_json);
409
 
562 mateuszvis 410
exit(0);
411
 
412
?>