# 2017-Sep-10. Created by Maxim T. 
$scriptPath = Split-Path -parent $MyInvocation.MyCommand.Definition; 
. ($scriptPath + "\psFunctions.ps1");     # Adding script with reusable functions
. ($scriptPath + "\psSetVariables.ps1");  # Adding script to assign values to common variables

$logFile        = $scriptPath + "\Log\" + $MyInvocation.MyCommand.Name.Replace(".ps1",".txt"); 
(Get-Date).ToString("HH:mm:ss") + " --- Starting script " + $MyInvocation.MyCommand.Name | Out-File $logFile -Encoding OEM; # starting logging to file.
$logSummary = (Get-Date).ToString("HH:mm:ss") + " Script: AlphaVantage Hist".PadRight(28);
$quoteIDFile = $dataRootFolder + "\QuotesIntraDay\AlphaVantageIntraday.txt"; if (Test-Path $quoteIDFile) { Remove-Item $quoteIDFile;} # Removing intraday file before each load

$listStart = $config.IndexOf("<AlphaVantage>"); $listEnd = $config.IndexOf("</AlphaVantage>"); 
if ($listStart -eq -1 -or $listEnd -eq -1 -or $listStart+1 -ge $listEnd) {"<AlphaVantage> Symbol list is empty. Exiting script." | Out-File $logFile -Encoding OEM -Append; exit(1);}
$symbolList = @($config | Select-Object -Index(($listStart+1)..($listEnd-1))); #list of symbols we will work on
"Symbol count: " + $symbolList.count + ". MinDate: $minDate" | Out-File $logFile -Encoding OEM -Append;

#https://www.alphavantage.co/query?function=TIME_SERIES_DAILY&symbol=HD&apikey=demo&datatype=csv&outputsize=compact (compact comes back with 4 months of data, full 
$alphaVantageKey=""; $alphaVantageKey = ($config | Select-Object -Index(($config.IndexOf("<AlphaVantageKey>"))+1)).Replace("</AlphaVantageKey>","");
if ($alphaVantageKey.Length -lt 4 -or $alphaVantageKey.Length -gt 50)  {$str="*** Error. AlphaVantageKey in psconfig file length is not between 4 and 50. Terminating script"; $str | Out-File $logFile -Encoding OEM -Append; $str; exit(1);}

$urlBase = "https://www.alphavantage.co/query?function=TIME_SERIES_DAILY&symbol=@Symbol@&apikey=@APIKey@&datatype=csv&outputsize=@OutputSize@"
$urlBase = $urlBase.Replace("@APIKey@", $alphaVantageKey); $reqRowsT = 0; $symbolCount=0;

ForEach($sLine in $symbolList) { # For each symbol
    $ret = GetSymbolInfo $sLine $quotesFolder $minDate; $symbol = $ret[0]; $nextDate = $ret[1]; $symbolMaxDate = $ret[2]; $symbolMaxDateAdj = $ret[3]; $symbolQuoteFile = $ret[4]; $lastQuote = $ret[5];
    "Symbol: $symbol. Next date: $nextDate. Quote file: $symbolQuoteFile" | Out-File $logFile -Encoding OEM -Append;
    if ($nextDate -gt $symbolMaxDate) { "  We already have data up to maximum configuration date of $symbolMaxDate. Will not request new data for this symbol." | Out-File $logFile -Encoding OEM -Append; continue; } 
    $outputSize = "full"; if (((get-date) - (get-date $nextDate)).Days -le 20) {$outputSize="compact";}

    # ========== Get quotes from website - Start ==============
    $url = $urlBase.Replace("@Symbol@",$symbol).Replace("@OutputSize@",$outputSize);
    "  Requesting url: " + $url | Out-File $logFile -Encoding OEM -Append; $webpage = ""; $reqRows=0; 
    $wc = new-object system.net.WebClient; $reqCount++; $symbolCount++;
    if ($symbolCount%4 -eq 0) {Start-Sleep 90;} # Delay 90 seconds after each 5 symbol requests. https://www.alpha-vantage.community/post/problems-with-api-call-frequency-9773234
    try {$webpage = $wc.DownloadData($url); } # Get page from url. This page will contain all quotes for single Symbol
    catch { $reqFailed++; "  " + $symbol + " - Not Found (web err) `r`n" | Out-File $logFile -Encoding OEM -Append; continue; } # if attempt to get webpage failed go to next symbol
    $quotesTxt = [System.Text.Encoding]::ASCII.GetString($webpage); # This variable now contains downloaded quotes:     timestamp,open,high,low,close,volume | 2017-09-12,159.0600,160.9300,158.7100,159.9100,5776559
    if ($quotesTxt.Contains("Error Message")) {" ***Error from website: " + $quotesTxt | Out-File $logFile -Encoding OEM -Append; continue;}
    if ($quotesTxt.length -le 45) { $reqFailed++; "  " + $symbol + " - Not Found (return empty file) `r`n" | Out-File $logFile -Encoding OEM -Append;  continue;} # Check if data received makes sense. If request size is less than 45 bytes (header length is 42), then there is something wrong with data received. Ignoring it, going to next symbol
    if ($quotesTxt.Contains("<html>")) {$reqFailed++; "  " + $symbol + " - Not Found (return html) `r`n" | Out-File $logFile -Encoding OEM -Append; continue;} # Result is html file, ignore such file
    $reqSucceed++; # If reached this point, that means symbol request was successful, go next to copy received data into local file
    # ========== Get quotes from website - End. Quote is in $quotesTxt ==============

    $ql = $quotesTxt.Split("`n") | ? {$_.trim() -ne ""  -and ($_.StartsWith("1") -or $_.StartsWith("2"))} | Sort-Object; # Sometimes duplicate records could come back, need to sort records and not load duplicate values
    $quoteToday="";
    $ql |%{$a=$_.Split(","); if($a[0] -ge $nextDate -and $a[0] -le $symbolMaxDateAdj -and $a[0] -gt $lastQuote -and $a[4].Trim() -match "^[\d\.]+$") {if ($a[0] -ne $todayYMD) {$a[0]+","+$a[4].Trim()+","+$symbol} else {$quoteToday=$a[0]+","+$a[4]+","+$symbol}; $reqRows++; $lastQuote=$a[0]; } }  |
    Out-File $symbolQuoteFile -Encoding OEM -Append #taking just date,Close,Symbol
    if ($quoteToday.Length -ne 0) {$quoteToday | Out-File $quoteIDFile -Encoding OEM -Append;} # Todays quote is always added to intraday file

    if ($lastQuote -lt $nextDate) {"Done: $symbol. No new quotes found. `r`n" | Out-File $logFile -Encoding OEM -Append;} # No new records where loaded
    else {"Done: $symbol (from: $nextDate). Record count: $reqRows. Last quote: $lastQuote `r`n" | Out-File $logFile -Encoding OEM -Append; $reqRowsT+=$reqRows;}
}
$duration = (NEW-TIMESPAN -Start $startTime -End (Get-Date)).TotalSeconds.ToString("#,##0") + " sec.";
(Get-Date).ToString("HH:mm:ss") + " --- Finished. Quotes Requested/Succeed/Failed/Rows: $reqCount/$reqSucceed/$reqFailed/$reqRowsT. Duration: $duration`r`n" | Out-File $logFile -Encoding OEM -append;
$logSummary + ". Quotes Requested/Succeed/Failed/Rows: $reqCount/$reqSucceed/$reqFailed/$reqRowsT. Duration: $duration";
