# ######################################################
# # 2018-Sep-07. Created by Maxim T. . 
# URL: https://stooq.com/q/d/l/?s=cadusd&d1=20160102&d2=20170913&i=d
# ######################################################
$scriptPath = Split-Path -parent $MyInvocation.MyCommand.Definition; 
. ($scriptPath + "\psFunctions.ps1");     # Adding script with reusable functions
. ($scriptPath + "\psSetVariables.ps1");  # Adding script to assign values to common variables

$logFile        = $scriptPath + "\Log\" + $MyInvocation.MyCommand.Name.Replace(".ps1",".txt"); 
(Get-Date).ToString("HH:mm:ss") + " --- Starting script " + $MyInvocation.MyCommand.Name | Out-File $logFile -Encoding OEM; # starting logging to file.
$logSummary = (Get-Date).ToString("HH:mm:ss") + " Script: ExchRate Stooq".PadRight(28);
Get-ChildItem -Path $currExchIDFolder -Filter *Intraday.txt -File | foreach { $_.Delete()}; # Delete existing all files from intraday folder

$listStart = $config.IndexOf("<Currency>"); $listEnd = $config.IndexOf("</Currency>"); 
if ($listStart -eq -1 -or $listEnd -eq -1 -or $listStart+1 -ge $listEnd) {"<currency> Symbol list is empty. Exiting script." | Out-File $logFile -Encoding OEM -Append; exit(1);}
$currList = @($config | Select-Object -Index(($listStart+1)..($listEnd-1))); #list of symbols we will work on
$currCount= $currList.Count; "Currency count: $currCount. MinDate: $minDate" | Out-File $logFile -Encoding OEM -Append;
if ($currCount -le 1) {"Found just one currency, will not create currency exchange database"; exit(1);}

$urlBase = "https://stooq.com/q/d/l/?s=@@CurrFrom@@@@CurrTo@@&d1=@@DateFrom@@&d2=@@DateTo@@&i=d".Replace("@@DateTo@@", $todayYMD.Replace("-",""));
For($currFromIndex=0; $currFromIndex -lt $currCount; $currFromIndex++) {
    For($currToIndex=0; $currToIndex -lt $currCount; $currToIndex++) {
        if ($currList[$currFromIndex] -ne $currList[$currToIndex]) {
            $currFrom = $currList[$currFromIndex]; $currTo = $currList[$currToIndex]; 
            $ret = GetCurrExchInfo $currFrom $currTo $currExchFolder $minDate; $currExchFile = $ret[0]; $nextDate = $ret[1]; $lastQuote = $ret[2];
            "CurrExch. From: $currFrom -> To: $currTo. Next date: $nextDate. Quote file: $currExchFile" | Out-File $logFile -Encoding OEM -Append;

            $url = $urlBase.Replace("@@CurrFrom@@",$currFrom).Replace("@@CurrTo@@",$currTo).Replace("@@DateFrom@@", $nextDate.Replace("-",""));  "  Url: " + $url | Out-File $logFile -Encoding OEM -Append;
            $wcD = new-object system.net.WebClient; $webpageD = ""; $reqCount++; $reqRows = 0; $lastQuote = ""; 
            try {$webpageD = $wcD.DownloadData($url);} # Get page from url
            catch { $reqFailed++; "  " + $symbol + " - Not Found (web err) `r`n" | Out-File $logFile -Encoding OEM -Append; if ($verbose) {$log + " No new data received."}; continue; } # if attempt to get webpage failed go to next symbol

            $cfD = [System.Text.Encoding]::ASCII.GetString($webpageD); # This variable now contains downloaded currency rates   Date,Open,High,Low,Close | 2017-09-13,1.45753,1.45905,1.44642,1.44723
            if ($cfD.Contains("<html")) {$reqFailed++; "  No new data (returned html)" | Out-File $logFile -Encoding OEM -Append; if($verbose) {$log + " No new data received."}; continue; } # Result is html file, something went wrong, ignore result that was just returned
            if ($cfD.Contains("No data")) {$reqFailed++; "  No new data (returned html)" | Out-File $logFile -Encoding OEM -Append; if($verbose) {$log + " No new data received."}; continue; } # Result is "No data", ignore result that was just returned

            $reqSucceed++;
            # First line just quote header; Ignore rows that says "Holiday"; Ignore empty rows; Ignore rows with quote date before $DateFrom; Ignore quotes where rate is empty value.
            $clD = $cfD.Split("`n") | select -Skip 1| Where {$_ -notlike "*holiday*" -and $_ -like "2*"} | Where {$_ -ne ""} | Where {$_.split(",")[1].Trim() -ne ""} | Sort-Object;
            ($clD  | %{$a=$_.Split(",");if ($a[0] -ge $nextDate -and $a[4].Trim() -match "^[\d\.]+$") {$reqRows++; $reqRowsT++; $a[0] + "," + $a[4].Trim() + "," + $CurrFrom + "," + $CurrTo; $lastQuote=$a[0];} }) | 
                Out-File $currExchFile -Encoding OEM -Append;

            if ($reqRows -eq 0) {"  No new data found." | Out-File $logFile -Encoding OEM -Append; }
            else {"  New rec count " + $reqRows.ToString().PadLeft(4) + ". Last quote: $lastQuote" | Out-File $logFile -Encoding OEM -Append;};
        }
    }
}

$duration = (NEW-TIMESPAN -Start $startTime -End (Get-Date)).TotalSeconds.ToString("#,##0") + " sec.";
(Get-Date).ToString("HH:mm:ss") + " --- Finished. CurrExch Requested/Succeed/Failed/Rows: $reqCount/$reqSucceed/$reqFailed/$reqRowsT. Duration: $duration`r`n" | Out-File $logFile -Encoding OEM -append;
$logSummary + ". CurrExch Requested/Succeed/Failed/Rows: $reqCount/$reqSucceed/$reqFailed/$reqRowsT. Duration: $duration";
