Subversion Repositories SvarDOS

Rev

Rev 926 | Rev 936 | Go to most recent revision | Details | Compare with Previous | Last modification | View Log | RSS feed

Rev Author Line No. Line
562 mateuszvis 1
<?php /*
2
 
3
  SvarDOS repo index builder
4
  Copyright (C) Mateusz Viste 2012-2022
5
 
734 bttr 6
  buildidx computes an index json file for the SvarDOS repository.
673 mateusz.vi 7
  it must be executed pointing to a directory that stores packages (*.svp)
562 mateuszvis 8
  files. buildidx will generate the index file and save it into the package
9
  repository.
10
 
11
  requires php-zip
12
 
912 mateusz.vi 13
  21 feb 2022: buildidx collects categories looking at the dir layout of each package + improved version string parsing (replaced version_compare call by dos_version_compare)
775 mateusz.vi 14
  17 feb 2022: checking for non-8+3 filenames in packages and duplicates + devload no longer part of CORE
736 mateusz.vi 15
  16 feb 2022: added warning about overlong version strings and wild files location
719 mateusz.vi 16
  15 feb 2022: index is generated as json, contains all filenames and alt versions
673 mateusz.vi 17
  14 feb 2022: packages are expected to have the *.svp extension
650 mateusz.vi 18
  12 feb 2022: skip source packages from being processed (*.src.zip)
562 mateuszvis 19
  20 jan 2022: rewritten the code from ANSI C to PHP for easier maintenance
20
  13 feb 2021: 'title' LSM field is no longer looked after
21
  11 feb 2021: lsm headers are no longer checked, so it is compatible with the simpler lsm format used by SvarDOS
22
  13 jan 2021: removed the identification line, changed CRC32 to bsum, not creating the listing.txt file and stopped compressing index
23
  23 apr 2017: uncompressed index is no longer created, added CRC32 of zib (bin only) files, if present
24
  28 aug 2016: listing.txt is always written inside the repo dir (instead of inside current dir)
25
  27 aug 2016: accepting full paths to repos (starting with /...)
26
  07 dec 2013: rewritten buildidx in ANSI C89
27
  19 aug 2013: add a compressed version of the index file to repos (index.gz)
28
  22 jul 2013: creating a listing.txt file with list of packages
29
  18 jul 2013: writing the number of packaged into the first line of the lst file
30
  11 jul 2013: added a switch to 7za to make it case insensitive when extracting lsm files
31
  10 jul 2013: changed unzip calls to 7za (to handle cases when appinfo is compressed with lzma)
32
  04 feb 2013: added CRC32 support
33
  22 sep 2012: forked 1st version from FDUPDATE builder
34
*/
35
 
921 mateusz.vi 36
$PVER = "20220222";
562 mateuszvis 37
 
38
 
39
// computes the BSD sum of a file and returns it
40
function file2bsum($fname) {
41
  $result = 0;
42
 
43
  $fd = fopen($fname, 'rb');
44
  if ($fd === false) return(0);
45
 
46
  while (!feof($fd)) {
47
 
48
    $buff = fread($fd, 1024 * 1024);
49
 
563 mateuszvis 50
    $slen = strlen($buff);
51
    for ($i = 0; $i < $slen; $i++) {
562 mateuszvis 52
      // rotr
53
      $result = ($result >> 1) | ($result << 15);
54
      // add and truncate to 16 bits
563 mateuszvis 55
      $result += ord($buff[$i]);
562 mateuszvis 56
      $result &= 0xffff;
57
    }
58
  }
59
 
60
  fclose($fd);
61
  return($result);
62
}
63
 
64
 
912 mateusz.vi 65
// translates a version string into a array of integer values.
66
// Accepted formats follow:
67
//    300.12.1
68
//    1
69
//    12.2.34.2-4.5
70
//    1.2c
71
//    1.01 beta+3
72
//    2013-12-31
73
//    20220222 alpha
74
function vertoarr($verstr) {
75
  $subver = array(0,0,0,0);
76
 
77
  // switch string to lcase for easier processing and trim any leading or trailing white spaces
78
  $verstr = strtolower(trim($verstr));
79
 
80
  // replace all '-' and '/' characters to '.' (uniformization of sub-version parts delimiters)
81
  $verstr = strtr($verstr, '-/', '..');
82
 
83
  // is there a subversion value? (for example "+4" in "1.05+4")
84
  $i = strrpos($verstr, '+', 1);
85
  if ($i !== false) {
86
    // validate the svar-version is a proper integer
87
    $svarver = substr($verstr, $i + 1);
88
    if (! preg_match('/[1-9][0-9]*/', $svarver)) {
89
      return(false);
90
    }
91
    $subver[3] = intval($svarver); // set the +rev as a very minor item
92
    $verstr = substr($verstr, 0, $i);
93
  }
94
 
927 mateusz.vi 95
  // any occurence of alpha,beta,gamma,delta etc preceded by a digit should have a space separator added
96
  // example: "2.6.0pre9" becomes "2.6.0 pre9"
97
  $verstr = preg_replace('/([0-9])(alpha|beta|gamma|delta|pre|rc|patch)/', '$1 $2', $verstr);
98
 
99
  // same as above, but this time adding a trailing space separator
100
  // example: "2.6.0 pre9" becomes "2.6.0 pre 9"
101
  $verstr = preg_replace('/(alpha|beta|gamma|delta|pre|rc|patch)([0-9])/', '$1 $2', $verstr);
102
 
921 mateusz.vi 103
  // is the version ending with ' alpha', 'beta', etc?
922 mateusz.vi 104
  if (preg_match('/ (alpha|beta|gamma|delta|pre|rc|patch)( [0-9]{1,4}){0,1}$/', $verstr)) {
921 mateusz.vi 105
    // if there is a trailing beta-number, process it first
106
    if (preg_match('/ [0-9]{1,4}$/', $verstr)) {
107
      $i = strrpos($verstr, ' ');
108
      $subver[2] = intval(substr($verstr, $i + 1));
109
      $verstr = trim(substr($verstr, 0, $i));
110
    }
912 mateusz.vi 111
    $i = strrpos($verstr, ' ');
112
    $greek = substr($verstr, $i + 1);
113
    $verstr = trim(substr($verstr, 0, $i));
114
    if ($greek == 'alpha') {
921 mateusz.vi 115
      $subver[1] = 1;
912 mateusz.vi 116
    } else if ($greek == 'beta') {
921 mateusz.vi 117
      $subver[1] = 2;
920 mateusz.vi 118
    } else if ($greek == 'gamma') {
921 mateusz.vi 119
      $subver[1] = 3;
920 mateusz.vi 120
    } else if ($greek == 'delta') {
921 mateusz.vi 121
      $subver[1] = 4;
122
    } else if ($greek == 'pre') {
123
      $subver[1] = 5;
914 mateusz.vi 124
    } else if ($greek == 'rc') {
921 mateusz.vi 125
      $subver[1] = 6;
922 mateusz.vi 126
    } else if ($greek == 'patch') { // this is a POST-release version, as opposed to all above that are PRE-release versions
127
      $subver[1] = 99;
912 mateusz.vi 128
    } else {
129
      return(false);
130
    }
914 mateusz.vi 131
  } else {
922 mateusz.vi 132
    $subver[1] = 98; // one less than the 'patch' level
912 mateusz.vi 133
  }
134
 
135
  // does the version string have a single-letter subversion? (1.0c)
136
  if (preg_match('/[a-z]$/', $verstr)) {
921 mateusz.vi 137
    $subver[0] = ord(substr($verstr, -1));
912 mateusz.vi 138
    $verstr = substr_replace($verstr, '', -1); // remove last character from string
139
  }
140
 
926 mateusz.vi 141
  // convert "30-jan-99", "1999-jan-30" and "30-jan-1999" versions to "30jan99" or "30jan1999"
142
  // note that dashes have already been replaced by dots
143
  if (preg_match('/^([0-9][0-9]){1,2}\.(jan|feb|mar|apr|may|jun|jul|aug|sep|oct|nov|dec)\.([0-9][0-9]){1,2}$/', $verstr)) {
144
    $verstr = str_replace('.', '', $verstr);
145
  }
146
 
147
  // convert "2009mar17" versions to "17mar2009"
148
  if (preg_match('/^[0-9]{4}(jan|feb|mar|apr|may|jun|jul|aug|sep|oct|nov|dec)[0-9]{2}$/', $verstr)) {
149
    $dy = substr($verstr, 7);
150
    $mo = substr($verstr, 4, 3);
151
    $ye = substr($verstr, 0, 4);
925 mateusz.vi 152
    $verstr = "{$dy}{$mo}{$ye}";
923 mateusz.vi 153
  }
154
 
155
  // convert "30jan99" versions to 99.1.30 and "30jan1999" to 1999.1.30
925 mateusz.vi 156
  if (preg_match('/^[0-3][0-9](jan|feb|mar|apr|may|jun|jul|aug|sep|oct|nov|dec)([0-9][0-9]){1,2}$/', $verstr)) {
923 mateusz.vi 157
    $months = array('jan' => 1, 'feb' => 2, 'mar' => 3, 'apr' => 4, 'may' => 5, 'jun' => 6, 'jul' => 7, 'aug' => 8, 'sep' => 9, 'oct' => 10, 'nov' => 11, 'dec' => 12);
158
    $dy = substr($verstr, 0, 2);
159
    $mo = $months[substr($verstr, 2, 3)];
160
    $ye = substr($verstr, 5);
161
    $verstr = "{$ye}.{$mo}.{$dy}";
162
  }
163
 
912 mateusz.vi 164
  // validate the format is supported, should be something no more complex than 1.05.3.33
919 mateusz.vi 165
  if (! preg_match('/^[0-9][0-9.]{0,20}$/', $verstr)) {
912 mateusz.vi 166
    return(false);
167
  }
168
 
169
  // NOTE: a zero right after a separator and trailed with a digit (as in 1.01)
170
  //       has a special meaning
171
  $exploded = explode('.', $verstr);
172
  if (count($exploded) > 16) {
173
    return(false);
174
  }
921 mateusz.vi 175
  $exploded[16] = $subver[0]; // a-z (1.0c)
176
  $exploded[17] = $subver[1]; // alpha/beta/gamma/delta/rc/pre
177
  $exploded[18] = $subver[2]; // alpha-beta-gamma subversion (eg. "beta 9")
912 mateusz.vi 178
  $exploded[19] = $subver[3]; // svar-ver (1.0+5)
179
  for ($i = 0; $i < 20; $i++) if (empty($exploded[$i])) $exploded[$i] = '0';
180
 
181
  ksort($exploded);
182
 
183
  return($exploded);
184
}
185
 
186
 
187
function dos_version_compare($v1, $v2) {
188
  $v1arr = vertoarr($v1);
189
  $v2arr = vertoarr($v2);
190
  for ($i = 0; $i < count($v1arr); $i++) {
921 mateusz.vi 191
    if ($v1arr[$i] > $v2arr[$i]) return(1);
192
    if ($v1arr[$i] < $v2arr[$i]) return(-1);
912 mateusz.vi 193
  }
194
  return(0);
195
}
196
 
197
 
562 mateuszvis 198
// reads file fil from zip archive z and returns its content, or false on error
199
function read_file_from_zip($z, $fil) {
200
  $zip = new ZipArchive;
201
  if ($zip->open($z, ZipArchive::RDONLY) !== true) {
202
    echo "ERROR: failed to open zip file '{$z}'\n";
203
    return(false);
204
  }
205
 
206
  // load the appinfo/pkgname.lsm file
207
  $res = $zip->getFromName($fil, 8192, ZipArchive::FL_NOCASE);
208
 
209
  $zip->close();
210
  return($res);
211
}
212
 
213
 
731 mateusz.vi 214
function read_list_of_files_in_zip($z) {
215
  $zip = new ZipArchive;
216
  if ($zip->open($z, ZipArchive::RDONLY) !== true) {
217
    echo "ERROR: failed to open zip file '{$z}'\n";
218
    return(false);
219
  }
220
 
221
  $res = array();
222
  for ($i = 0; $i < $zip->numFiles; $i++) $res[] = $zip->getNameIndex($i);
223
 
224
  $zip->close();
225
  return($res);
226
}
227
 
228
 
562 mateuszvis 229
// reads a LSM string and returns it in the form of an array
230
function parse_lsm($s) {
231
  $res = array();
232
  for ($l = strtok($s, "\n"); $l !== false; $l = strtok("\n")) {
233
    // the line is "token: value", let's find the colon
234
    $colpos = strpos($l, ':');
235
    if (($colpos === false) || ($colpos === 0)) continue;
236
    $tok = strtolower(trim(substr($l, 0, $colpos)));
237
    $val = trim(substr($l, $colpos + 1));
238
    $res[$tok] = $val;
239
  }
240
  return($res);
241
}
242
 
243
 
731 mateusz.vi 244
// on PHP 8+ there is str_starts_with(), but not on PHP 7 so I use this
245
function str_head_is($haystack, $needle) {
246
  return strpos($haystack, $needle) === 0;
247
}
248
 
249
 
791 mateusz.vi 250
// returns an array that contains CORE packages (populated from the core subdirectory in pkgdir)
251
function load_core_list($repodir) {
252
  $res = array();
253
 
254
  foreach (scandir($repodir . '/core/') as $f) {
255
    if (!preg_match('/\.svp$/', $f)) continue;
256
    $res[] = explode('.', $f)[0];
257
  }
258
  return($res);
259
}
260
 
261
 
562 mateuszvis 262
// ***************** MAIN ROUTINE *********************************************
263
 
719 mateusz.vi 264
//echo "SvarDOS repository index generator ver {$PVER}\n";
562 mateuszvis 265
 
266
if (($_SERVER['argc'] != 2) || ($_SERVER['argv'][1][0] == '-')) {
267
  echo "usage: php buildidx.php repodir\n";
268
  exit(1);
269
}
270
 
271
$repodir = $_SERVER['argv'][1];
272
 
273
$pkgfiles = scandir($repodir);
274
$pkgcount = 0;
275
 
738 mateusz.vi 276
 
795 mateusz.vi 277
// load the list of CORE and MSDOS_COMPAT packages
738 mateusz.vi 278
 
791 mateusz.vi 279
$core_packages_list = load_core_list($repodir);
804 bttr 280
$msdos_compat_list = explode(' ', 'append assign attrib chkdsk choice command comp cpidos debug defrag deltree diskcomp diskcopy display edit edlin exe2bin fc fdapm fdisk find format help himemx kernel keyb label localcfg mem mirror mode more move nlsfunc print replace share shsucdx sort swsubst tree undelete unformat xcopy');
738 mateusz.vi 281
 
719 mateusz.vi 282
// do a list of all svp packages with their available versions and descriptions
562 mateuszvis 283
 
719 mateusz.vi 284
$pkgdb = array();
285
foreach ($pkgfiles as $fname) {
801 mateusz.vi 286
  if (!preg_match('/\.svp$/i', $fname)) continue; // skip non-svp files
562 mateuszvis 287
 
801 mateusz.vi 288
  if (!preg_match('/^[a-zA-Z0-9+. _-]*\.svp$/', $fname)) {
289
    echo "ERROR: {$fname} has a very weird name\n";
290
    continue;
291
  }
292
 
719 mateusz.vi 293
  $path_parts = pathinfo($fname);
294
  $pkgnam = explode('-', $path_parts['filename'])[0];
295
  $pkgfullpath = realpath($repodir . '/' . $fname);
562 mateuszvis 296
 
719 mateusz.vi 297
  $lsm = read_file_from_zip($pkgfullpath, "appinfo/{$pkgnam}.lsm");
562 mateuszvis 298
  if ($lsm == false) {
802 mateusz.vi 299
    echo "ERROR: {$fname} does not contain an LSM file at the expected location\n";
719 mateusz.vi 300
    continue;
562 mateuszvis 301
  }
302
  $lsmarray = parse_lsm($lsm);
303
  if (empty($lsmarray['version'])) {
719 mateusz.vi 304
    echo "ERROR: lsm file in {$fname} does not contain a version\n";
305
    continue;
562 mateuszvis 306
  }
730 mateusz.vi 307
  if (strlen($lsmarray['version']) > 16) {
737 mateusz.vi 308
    echo "ERROR: version string in lsm file of {$fname} is too long (16 chars max)\n";
730 mateusz.vi 309
    continue;
310
  }
562 mateuszvis 311
  if (empty($lsmarray['description'])) {
719 mateusz.vi 312
    echo "ERROR: lsm file in {$fname} does not contain a description\n";
313
    continue;
562 mateuszvis 314
  }
315
 
731 mateusz.vi 316
  // validate the files present in the archive
317
  $listoffiles = read_list_of_files_in_zip($pkgfullpath);
739 mateusz.vi 318
  $pkgdir = $pkgnam;
319
 
768 mateusz.vi 320
  // special rule for "parent and children" packages
321
  if (str_head_is($pkgnam, 'djgpp_')) $pkgdir = 'djgpp'; // djgpp_* packages put their files in djgpp
754 mateusz.vi 322
  if ($pkgnam == 'fbc_help') $pkgdir = 'fbc'; // FreeBASIC help goes to the FreeBASIC dir
802 mateusz.vi 323
  if ($pkgnam == 'clamdb') $pkgdir = 'clamav'; // data patterns for clamav
739 mateusz.vi 324
 
768 mateusz.vi 325
  // array used to detect duplicated entries after lower-case conversion
326
  $duparr = array();
327
 
909 mateusz.vi 328
  // will hold the list of categories that this package belongs to
329
  $catlist = array();
330
 
731 mateusz.vi 331
  foreach ($listoffiles as $f) {
332
    $f = strtolower($f);
768 mateusz.vi 333
    $path_array = explode('/', $f);
334
    // emit a warning when non-8+3 filenames are spotted and find duplicates
335
    foreach ($path_array as $item) {
336
      if (empty($item)) continue; // skip empty items at end of paths (eg. appinfo/)
337
      if (!preg_match("/[a-z0-9!#$%&'()@^_`{}~-]{1,8}(\.[a-z0-9!#$%&'()@^_`{}~-]{1,3}){0,1}/", $item)) {
338
        echo "WARNING: {$fname} contains a non-8+3 path (or weird char): {$item} (in $f)\n";
339
      }
340
    }
341
    // look for dups
342
    if (array_search($f, $duparr) !== false) {
343
      echo "WARNING: {$fname} contains a duplicated entry: '{$f}'\n";
344
    } else {
345
      $duparr[] = $f;
346
    }
731 mateusz.vi 347
    // LSM file is ok
348
    if ($f === "appinfo/{$pkgnam}.lsm") continue;
349
    if ($f === "appinfo/") continue;
795 mateusz.vi 350
    // CORE and MSDOS_COMPAT packages are premium citizens and can do a little more
909 mateusz.vi 351
    $core_or_msdoscompat = 0;
352
    if (array_search($pkgnam, $core_packages_list) !== false) {
353
      $catlist[] = 'core';
354
      $core_or_msdoscompat = 1;
355
    }
356
    if (array_search($pkgnam, $msdos_compat_list) !== false) {
357
      $catlist[] = 'msdos_compat';
358
      $core_or_msdoscompat = 1;
359
    }
360
    if ($core_or_msdoscompat == 1) {
736 mateusz.vi 361
      if (str_head_is($f, 'bin/')) continue;
779 mateusz.vi 362
      if (str_head_is($f, 'cpi/')) continue;
749 mateusz.vi 363
      if (str_head_is($f, "doc/{$pkgdir}/")) continue;
364
      if ($f === 'doc/') continue;
365
      if (str_head_is($f, "nls/{$pkgdir}.")) continue;
366
      if ($f === 'nls/') continue;
736 mateusz.vi 367
    }
798 mateusz.vi 368
    // the help package is allowed to put files in... help
369
    if (($pkgnam == 'help') && (str_head_is($f, 'help/'))) continue;
909 mateusz.vi 370
    // must be category-prefixed file, add it to the list of categories for this package
371
    $catlist[] = explode('/', $f)[0];
749 mateusz.vi 372
    // well-known "category" dirs are okay
739 mateusz.vi 373
    if (str_head_is($f, "progs/{$pkgdir}/")) continue;
731 mateusz.vi 374
    if ($f === 'progs/') continue;
739 mateusz.vi 375
    if (str_head_is($f, "devel/{$pkgdir}/")) continue;
731 mateusz.vi 376
    if ($f === 'devel/') continue;
739 mateusz.vi 377
    if (str_head_is($f, "games/{$pkgdir}/")) continue;
731 mateusz.vi 378
    if ($f === 'games/') continue;
739 mateusz.vi 379
    if (str_head_is($f, "drivers/{$pkgdir}/")) continue;
731 mateusz.vi 380
    if ($f === 'drivers/') continue;
768 mateusz.vi 381
    echo "WARNING: {$fname} contains a file in an illegal location: {$f}\n";
731 mateusz.vi 382
  }
383
 
912 mateusz.vi 384
  // do I understand the version string?
385
  if (vertoarr($lsmarray['version']) === false) echo "WARNING: {$fname} parsing of version string failed ('{$lsmarray['version']}')\n";
386
 
719 mateusz.vi 387
  $meta['fname'] = $fname;
388
  $meta['desc'] = $lsmarray['description'];
909 mateusz.vi 389
  $meta['cats'] = array_unique($catlist);
719 mateusz.vi 390
 
391
  $pkgdb[$pkgnam][$lsmarray['version']] = $meta;
392
}
393
 
801 mateusz.vi 394
 
719 mateusz.vi 395
$db = array();
909 mateusz.vi 396
$cats = array();
719 mateusz.vi 397
 
909 mateusz.vi 398
// ******** compute the version-sorted list of packages with a single *********
399
// ******** description and category list for each package ********************
400
 
719 mateusz.vi 401
// iterate over each svp package
402
foreach ($pkgdb as $pkg => $versions) {
403
 
404
  // sort filenames by version, highest first
912 mateusz.vi 405
  uksort($versions, "dos_version_compare");
719 mateusz.vi 406
  $versions = array_reverse($versions, true);
407
 
408
  foreach ($versions as $ver => $meta) {
409
    $fname = $meta['fname'];
410
    $desc = $meta['desc'];
411
 
412
    $bsum = file2bsum(realpath($repodir . '/' . $fname));
413
 
414
    $meta2['ver'] = strval($ver);
415
    $meta2['bsum'] = $bsum;
416
 
417
    if (empty($db[$pkg]['desc'])) $db[$pkg]['desc'] = $desc;
909 mateusz.vi 418
    if (empty($db[$pkg]['cats'])) {
419
      $db[$pkg]['cats'] = $meta['cats'];
420
      $cats = array_unique(array_merge($cats, $meta['cats']));
421
    }
719 mateusz.vi 422
    $db[$pkg]['versions'][$fname] = $meta2;
423
  }
424
 
562 mateuszvis 425
  $pkgcount++;
426
 
427
}
428
 
719 mateusz.vi 429
if ($pkgcount < 100) echo "WARNING: an unexpectedly low number of packages has been found in the repo ({$pkgcount})\n";
562 mateuszvis 430
 
801 mateusz.vi 431
$json_blob = json_encode($db);
432
if ($json_blob === false) {
433
  echo "ERROR: JSON convertion failed! -> ";
434
  switch (json_last_error()) {
435
    case JSON_ERROR_DEPTH:
436
      echo 'maximum stack depth exceeded';
437
      break;
438
    case JSON_ERROR_STATE_MISMATCH:
439
      echo 'underflow of the modes mismatch';
440
      break;
441
    case JSON_ERROR_CTRL_CHAR:
442
      echo 'unexpected control character found';
443
      break;
444
    case JSON_ERROR_UTF8:
445
      echo 'malformed utf-8 characters';
446
      break;
447
    default:
448
      echo "unknown error";
449
      break;
450
  }
451
  echo "\n";
452
}
453
 
909 mateusz.vi 454
file_put_contents($repodir . '/_index.json', $json_blob);
562 mateuszvis 455
 
909 mateusz.vi 456
$cats_json = json_encode($cats);
457
file_put_contents($repodir . '/_cats.json', $cats_json);
458
 
562 mateuszvis 459
exit(0);
460
 
461
?>