WebSVN – SvarDOS – Diff – /buildidx/buildidx.php


<?php /*
 
  SvarDOS repo index builder
  Copyright (C) Mateusz Viste 2012-2022
 
  buildidx computes an index tsv file for the SvarDOS repository.
  it must be executed pointing to a directory that stores packages (*.svp)
  files. buildidx will generate the index file and save it into the package
  repository.
 
  requires php-zip
 
  16 feb 2022: added warning about overlong version strings
  15 feb 2022: index is generated as json, contains all filenames and alt versions
  14 feb 2022: packages are expected to have the *.svp extension
  12 feb 2022: skip source packages from being processed (*.src.zip)
  20 jan 2022: rewritten the code from ANSI C to PHP for easier maintenance
  13 feb 2021: 'title' LSM field is no longer looked after
  11 feb 2021: lsm headers are no longer checked, so it is compatible with the simpler lsm format used by SvarDOS
  13 jan 2021: removed the identification line, changed CRC32 to bsum, not creating the listing.txt file and stopped compressing index
  23 apr 2017: uncompressed index is no longer created, added CRC32 of zib (bin only) files, if present
  28 aug 2016: listing.txt is always written inside the repo dir (instead of inside current dir)
  27 aug 2016: accepting full paths to repos (starting with /...)
  07 dec 2013: rewritten buildidx in ANSI C89
  19 aug 2013: add a compressed version of the index file to repos (index.gz)
  22 jul 2013: creating a listing.txt file with list of packages
  18 jul 2013: writing the number of packaged into the first line of the lst file
  11 jul 2013: added a switch to 7za to make it case insensitive when extracting lsm files
  10 jul 2013: changed unzip calls to 7za (to handle cases when appinfo is compressed with lzma)
  04 feb 2013: added CRC32 support
  22 sep 2012: forked 1st version from FDUPDATE builder
*/
 
$PVER = "20220216";
 
 
// computes the BSD sum of a file and returns it
function file2bsum($fname) {
  $result = 0;
 
  $fd = fopen($fname, 'rb');
  if ($fd === false) return(0);
 
  while (!feof($fd)) {
 
    $buff = fread($fd, 1024 * 1024);
 
    $slen = strlen($buff);
    for ($i = 0; $i < $slen; $i++) {
      // rotr
      $result = ($result >> 1) | ($result << 15);
      // add and truncate to 16 bits
      $result += ord($buff[$i]);
      $result &= 0xffff;
    }
  }
 
  fclose($fd);
  return($result);
}
 
 
// reads file fil from zip archive z and returns its content, or false on error
function read_file_from_zip($z, $fil) {
  $zip = new ZipArchive;
  if ($zip->open($z, ZipArchive::RDONLY) !== true) {
    echo "ERROR: failed to open zip file '{$z}'\n";
    return(false);
  }
 
  // load the appinfo/pkgname.lsm file
  $res = $zip->getFromName($fil, 8192, ZipArchive::FL_NOCASE);
 
  $zip->close();
  return($res);
}
 
 
// reads a LSM string and returns it in the form of an array
function parse_lsm($s) {
  $res = array();
  for ($l = strtok($s, "\n"); $l !== false; $l = strtok("\n")) {
    // the line is "token: value", let's find the colon
    $colpos = strpos($l, ':');
    if (($colpos === false) || ($colpos === 0)) continue;
    $tok = strtolower(trim(substr($l, 0, $colpos)));
    $val = trim(substr($l, $colpos + 1));
    $res[$tok] = $val;
  }
  return($res);
}
 
 
// ***************** MAIN ROUTINE *********************************************
 
//echo "SvarDOS repository index generator ver {$PVER}\n";
 
if (($_SERVER['argc'] != 2) || ($_SERVER['argv'][1][0] == '-')) {
  echo "usage: php buildidx.php repodir\n";
  exit(1);
}
 
$repodir = $_SERVER['argv'][1];
 
$pkgfiles = scandir($repodir);
$pkgcount = 0;
 
// do a list of all svp packages with their available versions and descriptions
 
$pkgdb = array();
foreach ($pkgfiles as $fname) {
  if (!preg_match('/.svp$/i', $fname)) continue; // skip non-svp files
 
  $path_parts = pathinfo($fname);
  $pkgnam = explode('-', $path_parts['filename'])[0];
  $pkgfullpath = realpath($repodir . '/' . $fname);
 
  $lsm = read_file_from_zip($pkgfullpath, "appinfo/{$pkgnam}.lsm");
  if ($lsm == false) {
    echo "ERROR: pkg {$fname} does not contain an LSM file at the expected location\n";
    continue;
  }
  $lsmarray = parse_lsm($lsm);
  if (empty($lsmarray['version'])) {
    echo "ERROR: lsm file in {$fname} does not contain a version\n";
    continue;
  }
  if (strlen($lsmarray['version']) > 16) {
    echo "WARNING: version string in lsm file of {$fname} is too long (16 chars max)\n";
    continue;
  }
  if (empty($lsmarray['description'])) {
    echo "ERROR: lsm file in {$fname} does not contain a description\n";
    continue;
  }
 
  $meta['fname'] = $fname;
  $meta['desc'] = $lsmarray['description'];
 
  $pkgdb[$pkgnam][$lsmarray['version']] = $meta;
}
 
$db = array();
 
// iterate over each svp package
foreach ($pkgdb as $pkg => $versions) {
 
  // sort filenames by version, highest first
  uksort($versions, "version_compare");
  $versions = array_reverse($versions, true);
 
  foreach ($versions as $ver => $meta) {
    $fname = $meta['fname'];
    $desc = $meta['desc'];
 
    $bsum = file2bsum(realpath($repodir . '/' . $fname));
 
    $meta2['ver'] = strval($ver);
    $meta2['bsum'] = $bsum;
 
    if (empty($db[$pkg]['desc'])) $db[$pkg]['desc'] = $desc;
    $db[$pkg]['versions'][$fname] = $meta2;
  }
 
  $pkgcount++;
 
}
 
if ($pkgcount < 100) echo "WARNING: an unexpectedly low number of packages has been found in the repo ({$pkgcount})\n";
 
file_put_contents($repodir . '/_index.json', json_encode($db));
 
exit(0);
 
?>
 

Subversion Repositories SvarDOS

(root)/buildidx/buildidx.php @ 734 – Rev 719 → 730

Rev 719		Rev 730
1	`<?php /*`	1	`<?php /*`
2		2
3	`SvarDOS repo index builder`	3	`SvarDOS repo index builder`
4	`Copyright (C) Mateusz Viste 2012-2022`	4	`Copyright (C) Mateusz Viste 2012-2022`
5		5
6	`buildidx computes an index tsv file for the SvarDOS repository.`	6	`buildidx computes an index tsv file for the SvarDOS repository.`
7	`it must be executed pointing to a directory that stores packages (*.svp)`	7	`it must be executed pointing to a directory that stores packages (*.svp)`
8	`files. buildidx will generate the index file and save it into the package`	8	`files. buildidx will generate the index file and save it into the package`
9	`repository.`	9	`repository.`
10		10
11	`requires php-zip`	11	`requires php-zip`
12		12
-		13	`16 feb 2022: added warning about overlong version strings`
13	`15 feb 2022: index is generated as json, contains all filenames and alt versions`	14	`15 feb 2022: index is generated as json, contains all filenames and alt versions`
14	`14 feb 2022: packages are expected to have the *.svp extension`	15	`14 feb 2022: packages are expected to have the *.svp extension`
15	`12 feb 2022: skip source packages from being processed (*.src.zip)`	16	`12 feb 2022: skip source packages from being processed (*.src.zip)`
16	`20 jan 2022: rewritten the code from ANSI C to PHP for easier maintenance`	17	`20 jan 2022: rewritten the code from ANSI C to PHP for easier maintenance`
17	`13 feb 2021: 'title' LSM field is no longer looked after`	18	`13 feb 2021: 'title' LSM field is no longer looked after`
18	`11 feb 2021: lsm headers are no longer checked, so it is compatible with the simpler lsm format used by SvarDOS`	19	`11 feb 2021: lsm headers are no longer checked, so it is compatible with the simpler lsm format used by SvarDOS`
19	`13 jan 2021: removed the identification line, changed CRC32 to bsum, not creating the listing.txt file and stopped compressing index`	20	`13 jan 2021: removed the identification line, changed CRC32 to bsum, not creating the listing.txt file and stopped compressing index`
20	`23 apr 2017: uncompressed index is no longer created, added CRC32 of zib (bin only) files, if present`	21	`23 apr 2017: uncompressed index is no longer created, added CRC32 of zib (bin only) files, if present`
21	`28 aug 2016: listing.txt is always written inside the repo dir (instead of inside current dir)`	22	`28 aug 2016: listing.txt is always written inside the repo dir (instead of inside current dir)`
22	`27 aug 2016: accepting full paths to repos (starting with /...)`	23	`27 aug 2016: accepting full paths to repos (starting with /...)`
23	`07 dec 2013: rewritten buildidx in ANSI C89`	24	`07 dec 2013: rewritten buildidx in ANSI C89`
24	`19 aug 2013: add a compressed version of the index file to repos (index.gz)`	25	`19 aug 2013: add a compressed version of the index file to repos (index.gz)`
25	`22 jul 2013: creating a listing.txt file with list of packages`	26	`22 jul 2013: creating a listing.txt file with list of packages`
26	`18 jul 2013: writing the number of packaged into the first line of the lst file`	27	`18 jul 2013: writing the number of packaged into the first line of the lst file`
27	`11 jul 2013: added a switch to 7za to make it case insensitive when extracting lsm files`	28	`11 jul 2013: added a switch to 7za to make it case insensitive when extracting lsm files`
28	`10 jul 2013: changed unzip calls to 7za (to handle cases when appinfo is compressed with lzma)`	29	`10 jul 2013: changed unzip calls to 7za (to handle cases when appinfo is compressed with lzma)`
29	`04 feb 2013: added CRC32 support`	30	`04 feb 2013: added CRC32 support`
30	`22 sep 2012: forked 1st version from FDUPDATE builder`	31	`22 sep 2012: forked 1st version from FDUPDATE builder`
31	`*/`	32	`*/`
32		33
33	`$PVER = "20220215";`	34	`$PVER = "20220216";`
34		35
35		36
36	`// computes the BSD sum of a file and returns it`	37	`// computes the BSD sum of a file and returns it`
37	`function file2bsum($fname) {`	38	`function file2bsum($fname) {`
38	`$result = 0;`	39	`$result = 0;`
39		40
40	`$fd = fopen($fname, 'rb');`	41	`$fd = fopen($fname, 'rb');`
41	`if ($fd === false) return(0);`	42	`if ($fd === false) return(0);`
42		43
43	`while (!feof($fd)) {`	44	`while (!feof($fd)) {`
44		45
45	`$buff = fread($fd, 1024 * 1024);`	46	`$buff = fread($fd, 1024 * 1024);`
46		47
47	`$slen = strlen($buff);`	48	`$slen = strlen($buff);`
48	`for ($i = 0; $i < $slen; $i++) {`	49	`for ($i = 0; $i < $slen; $i++) {`
49	`// rotr`	50	`// rotr`
50	`$result = ($result >> 1) \| ($result << 15);`	51	`$result = ($result >> 1) \| ($result << 15);`
51	`// add and truncate to 16 bits`	52	`// add and truncate to 16 bits`
52	`$result += ord($buff[$i]);`	53	`$result += ord($buff[$i]);`
53	`$result &= 0xffff;`	54	`$result &= 0xffff;`
54	`}`	55	`}`
55	`}`	56	`}`
56		57
57	`fclose($fd);`	58	`fclose($fd);`
58	`return($result);`	59	`return($result);`
59	`}`	60	`}`
60		61
61		62
62	`// reads file fil from zip archive z and returns its content, or false on error`	63	`// reads file fil from zip archive z and returns its content, or false on error`
63	`function read_file_from_zip($z, $fil) {`	64	`function read_file_from_zip($z, $fil) {`
64	`$zip = new ZipArchive;`	65	`$zip = new ZipArchive;`
65	`if ($zip->open($z, ZipArchive::RDONLY) !== true) {`	66	`if ($zip->open($z, ZipArchive::RDONLY) !== true) {`
66	`echo "ERROR: failed to open zip file '{$z}'\n";`	67	`echo "ERROR: failed to open zip file '{$z}'\n";`
67	`return(false);`	68	`return(false);`
68	`}`	69	`}`
69		70
70	`// load the appinfo/pkgname.lsm file`	71	`// load the appinfo/pkgname.lsm file`
71	`$res = $zip->getFromName($fil, 8192, ZipArchive::FL_NOCASE);`	72	`$res = $zip->getFromName($fil, 8192, ZipArchive::FL_NOCASE);`
72		73
73	`$zip->close();`	74	`$zip->close();`
74	`return($res);`	75	`return($res);`
75	`}`	76	`}`
76		77
77		78
78	`// reads a LSM string and returns it in the form of an array`	79	`// reads a LSM string and returns it in the form of an array`
79	`function parse_lsm($s) {`	80	`function parse_lsm($s) {`
80	`$res = array();`	81	`$res = array();`
81	`for ($l = strtok($s, "\n"); $l !== false; $l = strtok("\n")) {`	82	`for ($l = strtok($s, "\n"); $l !== false; $l = strtok("\n")) {`
82	`// the line is "token: value", let's find the colon`	83	`// the line is "token: value", let's find the colon`
83	`$colpos = strpos($l, ':');`	84	`$colpos = strpos($l, ':');`
84	`if (($colpos === false) \|\| ($colpos === 0)) continue;`	85	`if (($colpos === false) \|\| ($colpos === 0)) continue;`
85	`$tok = strtolower(trim(substr($l, 0, $colpos)));`	86	`$tok = strtolower(trim(substr($l, 0, $colpos)));`
86	`$val = trim(substr($l, $colpos + 1));`	87	`$val = trim(substr($l, $colpos + 1));`
87	`$res[$tok] = $val;`	88	`$res[$tok] = $val;`
88	`}`	89	`}`
89	`return($res);`	90	`return($res);`
90	`}`	91	`}`
91		92
92		93
93	`// *************** MAIN ROUTINE *******************************************`	94	`// *************** MAIN ROUTINE *******************************************`
94		95
95	`//echo "SvarDOS repository index generator ver {$PVER}\n";`	96	`//echo "SvarDOS repository index generator ver {$PVER}\n";`
96		97
97	`if (($_SERVER['argc'] != 2) \|\| ($_SERVER['argv'][1][0] == '-')) {`	98	`if (($_SERVER['argc'] != 2) \|\| ($_SERVER['argv'][1][0] == '-')) {`
98	`echo "usage: php buildidx.php repodir\n";`	99	`echo "usage: php buildidx.php repodir\n";`
99	`exit(1);`	100	`exit(1);`
100	`}`	101	`}`
101		102
102	`$repodir = $_SERVER['argv'][1];`	103	`$repodir = $_SERVER['argv'][1];`
103		104
104	`$pkgfiles = scandir($repodir);`	105	`$pkgfiles = scandir($repodir);`
105	`$pkgcount = 0;`	106	`$pkgcount = 0;`
106		107
107	`// do a list of all svp packages with their available versions and descriptions`	108	`// do a list of all svp packages with their available versions and descriptions`
108		109
109	`$pkgdb = array();`	110	`$pkgdb = array();`
110	`foreach ($pkgfiles as $fname) {`	111	`foreach ($pkgfiles as $fname) {`
111	`if (!preg_match('/.svp$/i', $fname)) continue; // skip non-svp files`	112	`if (!preg_match('/.svp$/i', $fname)) continue; // skip non-svp files`
112		113
113	`$path_parts = pathinfo($fname);`	114	`$path_parts = pathinfo($fname);`
114	`$pkgnam = explode('-', $path_parts['filename'])[0];`	115	`$pkgnam = explode('-', $path_parts['filename'])[0];`
115	`$pkgfullpath = realpath($repodir . '/' . $fname);`	116	`$pkgfullpath = realpath($repodir . '/' . $fname);`
116		117
117	`$lsm = read_file_from_zip($pkgfullpath, "appinfo/{$pkgnam}.lsm");`	118	`$lsm = read_file_from_zip($pkgfullpath, "appinfo/{$pkgnam}.lsm");`
118	`if ($lsm == false) {`	119	`if ($lsm == false) {`
119	`echo "ERROR: pkg {$fname} does not contain an LSM file at the expected location\n";`	120	`echo "ERROR: pkg {$fname} does not contain an LSM file at the expected location\n";`
120	`continue;`	121	`continue;`
121	`}`	122	`}`
122	`$lsmarray = parse_lsm($lsm);`	123	`$lsmarray = parse_lsm($lsm);`
123	`if (empty($lsmarray['version'])) {`	124	`if (empty($lsmarray['version'])) {`
124	`echo "ERROR: lsm file in {$fname} does not contain a version\n";`	125	`echo "ERROR: lsm file in {$fname} does not contain a version\n";`
125	`continue;`	126	`continue;`
-		127	`}`
-		128	`if (strlen($lsmarray['version']) > 16) {`
-		129	`echo "WARNING: version string in lsm file of {$fname} is too long (16 chars max)\n";`
-		130	`continue;`
126	`}`	131	`}`
127	`if (empty($lsmarray['description'])) {`	132	`if (empty($lsmarray['description'])) {`
128	`echo "ERROR: lsm file in {$fname} does not contain a description\n";`	133	`echo "ERROR: lsm file in {$fname} does not contain a description\n";`
129	`continue;`	134	`continue;`
130	`}`	135	`}`
131		136
132	`$meta['fname'] = $fname;`	137	`$meta['fname'] = $fname;`
133	`$meta['desc'] = $lsmarray['description'];`	138	`$meta['desc'] = $lsmarray['description'];`
134		139
135	`$pkgdb[$pkgnam][$lsmarray['version']] = $meta;`	140	`$pkgdb[$pkgnam][$lsmarray['version']] = $meta;`
136	`}`	141	`}`
137		142
138	`$db = array();`	143	`$db = array();`
139		144
140	`// iterate over each svp package`	145	`// iterate over each svp package`
141	`foreach ($pkgdb as $pkg => $versions) {`	146	`foreach ($pkgdb as $pkg => $versions) {`
142		147
143	`// sort filenames by version, highest first`	148	`// sort filenames by version, highest first`
144	`uksort($versions, "version_compare");`	149	`uksort($versions, "version_compare");`
145	`$versions = array_reverse($versions, true);`	150	`$versions = array_reverse($versions, true);`
146		151
147	`foreach ($versions as $ver => $meta) {`	152	`foreach ($versions as $ver => $meta) {`
148	`$fname = $meta['fname'];`	153	`$fname = $meta['fname'];`
149	`$desc = $meta['desc'];`	154	`$desc = $meta['desc'];`
150		155
151	`$bsum = file2bsum(realpath($repodir . '/' . $fname));`	156	`$bsum = file2bsum(realpath($repodir . '/' . $fname));`
152		157
153	`$meta2['ver'] = strval($ver);`	158	`$meta2['ver'] = strval($ver);`
154	`$meta2['bsum'] = $bsum;`	159	`$meta2['bsum'] = $bsum;`
155		160
156	`if (empty($db[$pkg]['desc'])) $db[$pkg]['desc'] = $desc;`	161	`if (empty($db[$pkg]['desc'])) $db[$pkg]['desc'] = $desc;`
157	`$db[$pkg]['versions'][$fname] = $meta2;`	162	`$db[$pkg]['versions'][$fname] = $meta2;`
158	`}`	163	`}`
159		164
160	`$pkgcount++;`	165	`$pkgcount++;`
161		166
162	`}`	167	`}`
163		168
164	`if ($pkgcount < 100) echo "WARNING: an unexpectedly low number of packages has been found in the repo ({$pkgcount})\n";`	169	`if ($pkgcount < 100) echo "WARNING: an unexpectedly low number of packages has been found in the repo ({$pkgcount})\n";`
165		170
166	`file_put_contents($repodir . '/_index.json', json_encode($db));`	171	`file_put_contents($repodir . '/_index.json', json_encode($db));`
167		172
168	`exit(0);`	173	`exit(0);`
169		174
170	`?>`	175	`?>`
171		176