from selenium import webdriver
import urllib.request,os,datetime
from bs4 import BeautifulSoup
options = webdriver.ChromeOptions()
driver = webdriver.Chrome(executable_path=r'C:\chromedriver_win32\chromedriver.exe', chrome_options=options)
driver.implicitly_wait(10)
driver.get("https://mylink")
elems = driver.find_elements_by_css_selector("[href*=PublicInfoServlet]") #finding the weblinks(html doc) I need to edit and create new html docs
for elem in elems: #iterate through all the html weblinks found on the main webpage
abc=elem.get_attribute("href")
print(abc)
page = urllib.request.urlopen(abc)
soup = BeautifulSoup(page,'html.parser')
a=soup.find("div", {"id": "SpanPrint"}) #identify the html tag that needs to be used to create the required html document
efg = (abc.split("=", 1)[1])
hig=(efg.split('&', 1)[0])
f = open(str(hig)+'.html', 'w')
message=str(a)
f.write(message)
f.close()
# Change path to reflect file location
x = str(datetime.date.today())
b = str(datetime.datetime.now())
c = x[0:10]
d = b[11:19]
e = str(c + d).replace(':', '')
filename = 'mypath' +str(hig)+'.html' #saving the new doc at the required location.
os.rename('mypath'.html',
'mypath' +str(hig) + e + '.html')
driver.quit()
