# 2019-Feb-25. Created by Maxim T., Adjusted by x 2022-01-26

## Lines below might be changed by users. Any double quote inside the script should be "escaped", that is `" instead of ". 
## #############################################################################################################################################################################################
## <<<<<<<<<<<<<<<<<<<<<<<<<<<<<< This will need to be changed every time Yahoo changes where price is located. So added psConfig parameters where values can be changed without changing script
$startStr1 = "data-field=`"regularMarketPrice`" data-trend=`"none`" data-pricehint=`"2`" value=`""; $endStr1 = "`"";
$startStr2 = "data-field=`"regularMarketPrice`" data-trend=`"none`" data-pricehint=`"4`" value=`""; $endStr2 = "`"";
$startStr3 = ",`"currentPrice`":{`"raw`":";                                                         $endStr3 = ",";
## #############################################################################################################################################################################################



$scriptPath = Split-Path -parent $MyInvocation.MyCommand.Definition; 
. ($scriptPath + "\psFunctions.ps1");     # Adding script with reusable functions
. ($scriptPath + "\psSetVariables.ps1");  # Adding script to assign values to common variables

$logFile        = $scriptPath + "\Log\" + $MyInvocation.MyCommand.Name.Replace(".ps1",".txt"); 
(Get-Date).ToString("HH:mm:ss") + " --- Starting script " + $MyInvocation.MyCommand.Name | Out-File $logFile -Encoding OEM; # starting logging to file.
$logSummary = (Get-Date).ToString("HH:mm:ss") + " Script: Yahoo Intraday".PadRight(28);
$quoteIDFile = $dataRootFolder + "\QuotesIntraDay\YahooIntraday.txt"; if (Test-Path $quoteIDFile) { Remove-Item $quoteIDFile;} # Removing intraday file before each load

$symbolList = @();
$listStart = $config.IndexOf("<Yahoo>"); $listEnd = $config.IndexOf("</Yahoo>"); 
if ($listStart -eq -1 -or $listEnd -eq -1 -or $listStart+1 -ge $listEnd) {"<Yahoo> Symbol list is empty. " | Out-File $logFile -Encoding OEM -Append; exit(1);}
$symbolList += @($config | Select-Object -Index(($listStart+1)..($listEnd-1))); #list of symbols we will work on

$symbolList = @($symbolList | ?{$a=$_.split(","); ($a[3] -eq $null -or $a[3] -eq "" -or $a[3] -eq "Y") -and ($a[2] -eq "" -or $a[2] -eq $null)}); # by default if value is empty or y, then get intra-day quotes. Also MaxDate for symbol should not be specified
"Symbol count: " + $symbolList.count + ". MinDate: $minDate" | Out-File $logFile -Encoding OEM -Append;

$urlBase = "https://finance.yahoo.com/quote/@@Symbol@@";   $mths = @("Jan","Feb","Mar","Apr","May","Jun","Jul","Aug","Sep","Oct","Nov","Dec");

$logSymbolFolder = $scriptPath + "\Log\SymbolWebpage"; if (!(Test-Path $logSymbolFolder)) {New-Item $logSymbolFolder -type directory}; # setting up folder where text output per symbol will be temporary stored;

ForEach($sLine in $symbolList) { # For each symbol
    $ret = GetSymbolInfo $sLine $quotesFolder $minDate; $symbol = $ret[0]; $nextDate = $ret[1]; $symbolMaxDate = $ret[2]; $symbolMaxDateAdj = $ret[3]; $symbolQuoteFile = $ret[4]; $lastQuote = $ret[5];
    "Symbol: " + $symbol.PadLeft(12) + ". Next date: $nextDate. Quote file: $symbolQuoteFile" | Out-File $logFile -Encoding OEM -Append; 
    if ($nextDate -gt $symbolMaxDate) { "  We already have data up to maximum configuration date of $symbolMaxDate. Will not request new data for this symbol." | Out-File $logFile -Encoding OEM -Append; continue; } 

     # ========== Get latest quote from website - Start ==============
     $url = $urlBase.Replace("@@Symbol@@",$symbol); 
    "  Requesting url: " + $url | Out-File $logFile -Encoding OEM -Append; $webpage = ""; $reqRows=0; 
     $wc = new-object system.net.WebClient; $reqCount++;
     try {$webpage = $wc.DownloadData($url); } # Get page from url. This page will contain all quotes for single Symbol
     catch { $reqFailed++; "  " + $symbol + " - Not Found (web err) `r`n" | Out-File $logFile -Encoding OEM -Append; continue; } # if attempt to get webpage failed go to next symbol
     
     $quotesTxt = [System.Text.Encoding]::ASCII.GetString($webpage); $quotesTxt = $quotesTxt.Replace("???",""); # This variable now contains downloaded quotes
     $quotesTxt | Out-File ($logSymbolFolder + "\" + $symbol + ".txt") -Encoding OEM ; # Saving web page to \Log\SymbolWebpage\<Symbol>.txt file for review later (if needed)

    # ---------------------------------------------
    # Identifying price
    # ---------------------------------------------
    $priceTxt=""; 
    # attempting to find price between values in $startStr1 and $endStr1 strings
    $startStr = $startStr1; $endStr = $endStr1;
    if ($priceTxt -eq "") { # If price not identified YET
        if ($quotesTxt.Contains($startStr)) {
            $quotesTxt1 = $quotesTxt.Substring($quotesTxt.IndexOf($startStr) + $startStr.Length,500); # This now includes price and date
            if ($quotesTxt1.Contains($endStr)) {
                $priceTxt = $quotesTxt1.Substring(0,$quotesTxt1.IndexOf($endStr)).Replace(",",""); "  Found price: $priceTxt. Price found between strings '$startStr' and '$endStr'" | Out-File $logFile -Encoding OEM -Append; $reqSucceed++;
            } else {"  ** For Symbol: " + $symbol + " start string '$startStr' found but end string'$endStr' not found. `r`n" | Out-File $logFile -Encoding OEM -Append;}
        } else {"  ** For Symbol: " + $symbol + " start string '$startStr' not found. `r`n" | Out-File $logFile -Encoding OEM -Append;}
    };  

    # attempting to find price between values in $startStr2 and $endStr2 strings
    $startStr = $startStr2; $endStr = $endStr2;
    if ($priceTxt -eq "") { # If price not identified YET
        if ($quotesTxt.Contains($startStr)) {
            $quotesTxt1 = $quotesTxt.Substring($quotesTxt.IndexOf($startStr) + $startStr.Length,500); # This now includes price and date
            if ($quotesTxt1.Contains($endStr)) {
                $priceTxt = $quotesTxt1.Substring(0,$quotesTxt1.IndexOf($endStr)).Replace(",",""); "  Found price: $priceTxt. Price found between strings '$startStr' and '$endStr'" | Out-File $logFile -Encoding OEM -Append; $reqSucceed++;
            } else {"  ** For Symbol: " + $symbol + " start string '$startStr' found but end string'$endStr' not found. `r`n" | Out-File $logFile -Encoding OEM -Append;}
        } else {"  ** For Symbol: " + $symbol + " start string '$startStr' not found. `r`n" | Out-File $logFile -Encoding OEM -Append;}
    };  

    # attempting to find price between values in $startStr3 and $endStr3 strings
    $startStr = $startStr3; $endStr = $endStr3;
    if ($priceTxt -eq "") { # If price not identified YET
        if ($quotesTxt.Contains($startStr)) {
            $quotesTxt1 = $quotesTxt.Substring($quotesTxt.IndexOf($startStr) + $startStr.Length,500); # This now includes price and date
            if ($quotesTxt1.Contains($endStr)) {
                $priceTxt = $quotesTxt1.Substring(0,$quotesTxt1.IndexOf($endStr)).Replace(",",""); "  Found price: $priceTxt. Price found between strings '$startStr' and '$endStr'" | Out-File $logFile -Encoding OEM -Append; $reqSucceed++;
            } else {"  ** For Symbol: " + $symbol + " start string '$startStr' found but end string'$endStr' not found. `r`n" | Out-File $logFile -Encoding OEM -Append;}
        } else {"  ** For Symbol: " + $symbol + " start string '$startStr' not found. `r`n" | Out-File $logFile -Encoding OEM -Append;}
    };


    # This code check if at this point price is found, if not, report failure
    if ($priceTxt -eq "") { # If price not identified YET
         $reqFailed++; "  ** For Symbol: " + $symbol + " new quotes not found. `r`n" | Out-File $logFile -Encoding OEM -Append; 
         continue; # go to next symbol !!!!!!!!!!!!!!!!
    }# if that symbol has no quotes, then record errors
    
    $date = (Get-Date).ToString("yyyy-MM-dd"); # Assuming that always returns todays data. If no, you need to disable intraday requests for those symbols
    $dayOfWeek = ([datetime]::ParseExact($date,"yyyy-MM-dd",$null)).DayofWeek.ToString().Substring(0,3)
    if ($dayOfWeek -eq "Sat") {$date = (Get-Date).AddDays(-1).ToString("yyyy-MM-dd");}
    if ($dayOfWeek -eq "Sun") {$date = (Get-Date).AddDays(-2).ToString("yyyy-MM-dd");}

    if($date -ge $nextDate) { # If quote date is after or equal to date we already have
        $date + "," + $priceTxt + "," + $symbol | Out-File $quoteIDFile -Encoding OEM -Append; $reqRowsT++;
    }
}

$duration = (NEW-TIMESPAN -Start $startTime -End (Get-Date)).TotalSeconds.ToString("#,##0") + " sec.";
(Get-Date).ToString("HH:mm:ss") + " --- Finished. Quotes Requested/Succeed/Failed/Rows: $reqCount/$reqSucceed/$reqFailed/$reqRowsT. Duration: $duration`r`n" | Out-File $logFile -Encoding OEM -append;
$logSummary + ". Quotes Requested/Succeed/Failed/Rows: $reqCount/$reqSucceed/$reqFailed/$reqRowsT. Duration: $duration";

#$startStr = "<span class=`"Trsdu(0.3s) Fw(b) Fz(36px) Mb(-4px) D(ib)`" data-reactid=`"29`">"; $endStr = "</span>"; ## <<<<<<<<<<<<<<<<<<<<<<<<<<<<<< This will need to be changed every time Yahoo changes where price is located
#$startStr = "active=`"`" data-reactid=`"29`">"; $endStr = "</fin-streamer>"; 
## if parameters are set in psConfig, then use strings from psConfig
#if ($config.IndexOf("<YahooIntradayStartStr>") -ne -1) {$startStr = ($config | Select-Object -Index(($config.IndexOf("<YahooIntradayStartStr>"))+1)).Replace("</YahooIntradayStartStr>","");}
#if ($config.IndexOf("<YahooIntradayEndStr>") -ne -1)   {$endStr   = ($config | Select-Object -Index(($config.IndexOf("<YahooIntradayEndStr>"))+1)).Replace("</YahooIntradayEndStr>","");}
