nodejs_dist_list/parse_dist.php

   1 <?php
   2
   3 $baseurl  = "http://nodejs.org/dist";
   4
   5 // we scrape nodejs.org/dist site looking for versions that exist so we can construct a meta data database
   6 // so our lxc will be able to find versions in a more appropriate way
   7
   8 // first, get the main site
   9
  10 $fver = array();
  11 $fvern = 0;
  12 global $fver, $baseurl, $fvern;
  13
  14
  15 echo "Getting base list from $baseurl\n";
  16 $distbase = file_get_contents($baseurl);
  17
  18 // get all links that are directories
  19 preg_match_all('/<a [^>]*href="(.+)"/', $distbase, $firstlist);
  20
  21 $versions = array();
  22
  23 foreach($firstlist[1] as $chk) {
  24         //echo "Got: $chk\n";
  25         if(preg_match('/v[0-9]+.[0-9]+.[0-9]+\//', $chk)) {
  26                 if(preg_match('/v([0-9]+.[0-9]+.[0-9]+)\//', $chk, $vlist)) {
  27                         $versions[$vlist[1]] = $chk;
  28                 }
  29         }
  30 }
  31
  32 foreach($versions as $key => $val) {
  33         // now we get each version and parse whats in its directory
  34         echo "Checking $key => $baseurl/$val\n";
  35
  36         //if(preg_match("/0\.8\.[1].*/", $key)) {
  37                 $urlnow = "$baseurl/$val";
  38                 $vers_dir = file_get_contents($urlnow);
  39
  40                 preg_match_all('/<a [^>]*href="(.+)"/', $vers_dir, $fileslist);
  41
  42                 checkFilesList(rtrim($val, "/"), $key, $fileslist);
  43         //}
  44 }
  45
  46
  47 // done!
  48 //echo "and fver is now\n";
  49 //var_dump($fver);
  50
  51 echo "finding max versions\n";
  52 $maxstable[0] = 0;
  53 $maxstable[1] = 0;
  54 $maxstable[2] = 0;
  55 $maxdev[0] = 0;
  56 $maxdev[1] = 0;
  57 $maxdev[2] = 0;
  58 foreach($versions as $key => $val) {
  59         $vspl = explode(".", $key);
  60         if(($vspl[1]%2) == 1) {
  61                 // dev
  62                 if($vspl[0] >= $maxdev[0]) {
  63                         if($vspl[1] >= $maxdev[1]) {
  64                                 if($vspl[2] >= $maxdev[2]) {
  65                                         $maxdev[0] = $vspl[0];
  66                                         $maxdev[1] = $vspl[1];
  67                                         $maxdev[2] = $vspl[2];
  68                                 }
  69                         }
  70                 }
  71         } else {
  72                 // stable
  73                 if($vspl[0] >= $maxstable[0]) {
  74                         if($vspl[1] >= $maxstable[1]) {
  75                                 if($vspl[2] >= $maxstable[2]) {
  76                                         $maxstable[0] = $vspl[0];
  77                                         $maxstable[1] = $vspl[1];
  78                                         $maxstable[2] = $vspl[2];
  79                                 }
  80                         }
  81                 }
  82         }
  83 }
  84
  85 echo "getting init version\n";
  86 $initvers_l = explode("\n", file_get_contents("../lxc/init.js"));
  87 $initvers_t = preg_grep("/.*VERSION_FOR_PARSER.*/", $initvers_l);
  88 foreach($initvers_t as $val) {
  89         $cpl = explode(":", $val);
  90         $initvers = trim($cpl[1]);
  91 }
  92
  93 echo "dumping to versions file\n";
  94 $versdate = date("Ymd");
  95 $versfilename = "./versions/versions_file.$versdate";
  96 $versfile = fopen("$versfilename", "w");
  97 if($versfile !== false) {
  98         fwrite($versfile, "baseurl:$baseurl\n");
  99         foreach($fver as $line) {
 100                 fwrite($versfile, "$line\n");
 101         }
 102         fclose($versfile);
 103
 104         // compress teh file
 105         system("gzip -c versions/$versfilename > versions/$versfilename.gz");
 106
 107         // create a current versions file
 108         $cv = fopen("versions/current_version", "w");
 109         if($cv !== false) {
 110                 fwrite($cv, "version:$versdate\n");
 111                 fwrite($cv, "stable:".$maxstable[0].".".$maxstable[1].".".$maxstable[2]."\n");
 112                 fwrite($cv, "dev:".$maxdev[0].".".$maxdev[1].".".$maxdev[2]."\n");
 113                 fwrite($cv, "nodejs:$initvers\n");
 114                 fclose($cv);
 115         }
 116 } else {
 117         echo "Failed to open vers file!\n";
 118 }
 119
 120
 121 function checkFilesList($url, $vers, $fileslist) {
 122         global $fver, $baseurl, $fvern;
 123
 124         echo "Checking $url, $vers\n";
 125
 126         $shalist = array();
 127
 128         foreach($fileslist[1] as $fname) {
 129                 $is_echo = false;
 130                 $flurl = "";
 131                 $arch = "";
 132                 $plat = "";
 133                 $thissha = "";
 134
 135                 if($fname == "SHASUMS.txt") {
 136                         //$shas = file_get_
 137                         $shas_r = file_get_contents("$baseurl/$url/SHASUMS.txt");
 138                         $shalist = explode("\n", $shas_r);
 139                         foreach($shalist as $ssha) {
 140                                 $fsha = preg_split("/[ \t]+/", $ssha);
 141                                 if(isset($fsha[1])) $shalist[$fsha[1]] = $fsha[0];
 142                         }
 143                         //echo "Shasums file: $fname\n";
 144                         //var_dump($shalist);
 145                 } else if($fname == "MD5SUMS.txt") {
 146                         // nothing yet
 147                 } else  if(preg_match("/.*linux.*64\.tar\.gz/", $fname)) {
 148                         //echo "64 bit version for linux: $fname\n";
 149                         $thissha = "-";
 150                         if(isset($shalist[$fname])) $thissha = $shalist[$fname];
 151                         $is_echo = true;
 152                         $plat = "linux";
 153                         $arch = "x86_64";
 154                 } else if(preg_match("/.*linux.*86\.tar\.gz/", $fname)) {
 155                         //echo "32 bit version for linux: $fname\n";
 156                         $thissha = "-";
 157                         if(isset($shalist[$fname])) $thissha = $shalist[$fname];
 158                         //echo "$vers:$url/$fname:linux:i686:$thissha\n";
 159                         $is_echo = true;
 160                         $plat = "linux";
 161                         $arch = "i686";
 162                 } else  if(preg_match("/.*darwin.*86\.tar\.gz/", $fname)) {
 163                         //echo "32 bit version for darwin: $fname\n";
 164                         $thissha = "-";
 165                         if(isset($shalist[$fname])) $thissha = $shalist[$fname];
 166                         //echo "$vers:$url/$fname:darwin:i686:$thissha\n";
 167                         $is_echo = true;
 168                         $plat = "darwin";
 169                         $arch = "i686";
 170                 } else if(preg_match("/.*darwin.*64\.tar\.gz/", $fname)) {
 171                         //echo "64 bit version for darwin: $fname\n";
 172                         $thissha = "-";
 173                         if(isset($shalist[$fname])) $thissha = $shalist[$fname];
 174                         //echo "$vers:$url/$fname:darwin:x86_64:$thissha\n";
 175                         $is_echo = true;
 176                         $plat = "darwin";
 177                         $arch = "x86_64";
 178                 } else if(preg_match("/.*sunos.*64\.tar\.gz/", $fname)) {
 179                         // echo "64 bit version for sunos: $fname\n";
 180                         $thissha = "-";
 181                         if(isset($shalist[$fname])) $thissha = $shalist[$fname];
 182                         //echo "$vers:$url/$fname:sunos:x86_64:$thissha\n";
 183                         $is_echo = true;
 184                         $plat = "sunos";
 185                         $arch = "x86_64";
 186                 } else if(preg_match("/.*sunos.*86\.tar\.gz/", $fname)) {
 187                         //echo "32 bit version for sunos: $fname\n";
 188                         $thissha = "-";
 189                         if(isset($shalist[$fname])) $thissha = $shalist[$fname];
 190                         //echo "$vers:$url/$fname:sunos:i686:$thissha\n";
 191                         $is_echo = true;
 192                         $plat = "sunos";
 193                         $arch = "i686";
 194                 } else if(preg_match("/.*linux-arm.*\.tar\.gz/", $fname)) {
 195                         // echo "arm for linux: $fname\n";
 196                         $thissha = "-";
 197                         if(isset($shalist[$fname])) $thissha = $shalist[$fname];
 198                         //echo "$vers:$url/$fname:linux:armhf:$thissha\n";
 199                         $is_echo = true;
 200                         $plat = "linux";
 201                         $arch = "armhf";
 202                 } else if(preg_match("/node-v[0-9]+\.[0-9]+\.[0-9]+\.tar\.gz/", $fname)) {
 203                         $thissha = "-";
 204                         if(isset($shalist[$fname])) $thissha = $shalist[$fname];
 205                         //echo "$vers:$url/$fname:linux:armhf:$thissha\n";
 206                         $is_echo = true;
 207                         $plat = "src";
 208                         $arch = "-";
 209                 }
 210
 211                 if($is_echo) {
 212                         $fver[$fvern++] = "$vers:$plat:$arch:$url/$fname:$thissha";
 213                 }
 214         }
 215 }
 216 ?>