Lesson 14, Part 2: File Handling in Python

How to list out the the current directory name

import os
print('Path at terminal when executing this file')
print(os.getcwd() + '\n')

Path at terminal when executing this file
c:\Explore\SAS\Lesson14

%pwd

'c:\\Explore\\SAS\\Lesson14'

import os
os.environ['TEMP']

import os
for a in os.environ:
    print('Var: ',a, 'Value: ', os.getenv(a))
print ('all done')

import os for a in os.environ: print(a, os.getenv(a))

import os for a in os.environ: print('Var: ', a, 'Value: ', os.getenv(a)) print("all done")

How to list all files horizontally from a specified directory

import os
print(os.listdir("C:\Misc")) 

['County_data.sas', 'datasets_handler.py', 'Download_to_SDS_2018_9_4M5.sas', 'dump_data.py', 'Ex_Sum_statement_cars_data.sas', 'Generated_Code.sas', 'locate_datasets.py', 'MEPS_download_nonmacro.sas', 'Non_macro_code_2018.sas', 'resources.tar.gz', 'spd.sas', 'Stat6197_Spring_2020.sas', 'Subdir1', 'Subdir2', 'support.py', 'Sysfunc.sas', '__init__.py']

How to list all files vertically from a specified directory

import os
names = os.listdir('C:\Misc')
for p in names:
    print(p)

County_data.sas
datasets_handler.py
Download_to_SDS_2018_9_4M5.sas
dump_data.py
Ex_Sum_statement_cars_data.sas
Generated_Code.sas
locate_datasets.py
MEPS_download_nonmacro.sas
Non_macro_code_2018.sas
resources.tar.gz
spd.sas
Stat6197_Spring_2020.sas
Subdir1
Subdir2
support.py
Sysfunc.sas
__init__.py

import os
names = os.listdir("C:\Misc")
for p in names:
    print(p)

County_data.sas
datasets_handler.py
Download_to_SDS_2018_9_4M5.sas
dump_data.py
Ex_Sum_statement_cars_data.sas
Generated_Code.sas
locate_datasets.py
MEPS_download_nonmacro.sas
Non_macro_code_2018.sas
resources.tar.gz
spd.sas
Stat6197_Spring_2020.sas
Subdir1
Subdir2
support.py
Sysfunc.sas
__init__.py

How to list all files with a particular extension (.SAS) along with the root directory name (using the glob library)

The "raw" string treats the backslash as a regular character, not an escape character. (An escape character introduces an escape sequence, like \n for the newline character.

When using a regular Python string, it's necessary to use two backslashes in the file path because the first backslash is the escape character, but backslash + backslash effectively "unescapes" the character, producing a single backslash when you print it. (per email communication with Dolsy Smith, GW Library)

There's some useful explanation on this stackoverflow post.

path = r'C:\Misc\data'

import glob
path = 'C:\\Misc'
files = (f for f in glob.glob(path + '**/*.sas', recursive=True))
for f in files:
    print(f)

C:\Misc\County_data.sas
C:\Misc\Download_to_SDS_2018_9_4M5.sas
C:\Misc\Ex_Sum_statement_cars_data.sas
C:\Misc\Generated_Code.sas
C:\Misc\MEPS_download_nonmacro.sas
C:\Misc\Non_macro_code_2018.sas
C:\Misc\spd.sas
C:\Misc\Stat6197_Spring_2020.sas
C:\Misc\Sysfunc.sas

import glob
path = 'C:\\'
files = (f for f in glob.glob(path + '**/*sascfg_personal.py', recursive=True))
for f in files:
    print(f)

fd = open(r'C:\Users\fl\AppData\Local\anaconda3\Lib\site-packages\saspy\sascfg_personal.py')
print(fd.read())
fd.close()

import os
os.environ["PATH"] += ";C:\\Program Files\\SASHome\\SASFoundation\\9.4\\core\\sasext"

SAS_config_names=['winlocal']

default  = {'saspath'  : '/opt/sasinside/SASHome/SASFoundation/9.4/bin/sas_u8' }

winlocal = {'java'      : 'C:\Program Files (x86)\Common Files\Oracle\Java\javapath\java.exe',  'encoding'  : 'windows-1252'}

 

import glob
path = 'C:\\'
files = (f for f in glob.glob(path + '**/*ADAM*.sas', recursive=True))
for f in files:
    print(f)

C:\Data\ADAM.sas

import os
path = r'C:\Misc'
files = []
for r, d, f in os.walk(path):
    for file in f:
        if '.sas' in file:
            files.append(os.path.join(r, file))
            
for f in files:
    print(f)

C:\Misc\County_data.sas
C:\Misc\Download_to_SDS_2018_9_4M5.sas
C:\Misc\Ex_Sum_statement_cars_data.sas
C:\Misc\Generated_Code.sas
C:\Misc\MEPS_download_nonmacro.sas
C:\Misc\Non_macro_code_2018.sas
C:\Misc\spd.sas
C:\Misc\Stat6197_Spring_2020.sas
C:\Misc\Sysfunc.sas

How to list all subdirectories but no files (using the OS library)

import os
path = r'C:\Misc'
folders = []
for r, d, f in os.walk(path):
    for folder in d:
        folders.append(os.path.join(r, folder))
            
for f in folders:
    print(f)

C:\Misc\Subdir1
C:\Misc\Subdir2

How to list all files along with the parent directory name

from pathlib import Path
dir =  Path(r'C:\Misc')
files = dir.glob('*.sas')
for i in files:
    print(i)

C:\Misc\County_data.sas
C:\Misc\Download_to_SDS_2018_9_4M5.sas
C:\Misc\Ex_Sum_statement_cars_data.sas
C:\Misc\Generated_Code.sas
C:\Misc\MEPS_download_nonmacro.sas
C:\Misc\Non_macro_code_2018.sas
C:\Misc\spd.sas
C:\Misc\Stat6197_Spring_2020.sas
C:\Misc\Sysfunc.sas

How to list all files along with the parent directory name, and the date created but no names for subdirectories

import pandas as pd
from pathlib import Path
import time

p = Path(r'C:\Misc')
all_files = []
for i in p.rglob('*.SAS'):
    all_files.append((i.name, i.parent, time.ctime(i.stat().st_ctime)))

columns = ['File_Name','Parent', 'Created']
df = pd.DataFrame.from_records(all_files, columns=columns)
print(df.to_string(index=False))

                      File_Name   Parent                   Created
                County_data.sas  C:\Misc  Sun May 24 09:27:08 2020
 Download_to_SDS_2018_9_4M5.sas  C:\Misc  Fri Sep  4 16:54:12 2020
 Ex_Sum_statement_cars_data.sas  C:\Misc  Sun Apr 19 23:34:08 2020
             Generated_Code.sas  C:\Misc  Wed Sep 30 20:55:09 2020
     MEPS_download_nonmacro.sas  C:\Misc  Mon Jun 29 11:50:09 2020
        Non_macro_code_2018.sas  C:\Misc  Wed Sep 30 02:32:17 2020
                        spd.sas  C:\Misc  Sat Aug 31 15:06:16 2019
       Stat6197_Spring_2020.sas  C:\Misc  Sat May  9 10:41:12 2020
                    Sysfunc.sas  C:\Misc  Sat Aug 31 15:09:04 2019

from pathlib import Path
dir =  Path(r'C:\Data')
files = dir.glob('*.sas')
for i in files:
    print(i)

C:\Data\Change_FIPS_master.sas
C:\Data\gapminder.SAS
C:\Data\Merge_Data.sas
C:\Data\Read_county_data.sas

How to select rows and columns in Pandas using [ ], .loc, iloc, .at and .iat

from pathlib import Path
my_file = Path(r'C\Data\Create_formats.sas')
my_file.is_file() 

False

List out the name of the file if it exists in any folder(s) (using the OS library)

import os
path = 'C:\\'
files = []
for r, d, f in os.walk(path):
    for file in f:
        if 'gapminder.sas' in file:
            files.append(os.path.join(r, file))
            
for f in files:
    print(f)

C:\Data\gapminder.sas7bdat

List out the name of the file if it exists in any folder (using the glob library - more efficient and faster)

import glob
path = 'C:\\'
files = (f for f in glob.glob(path + '**/gapminder.sas', recursive=True))
for f in files:
    print(f)

C:\Data\gapminder.sas
C:\NIH\gapminder.sas
C:\Users\pmuhuri\Documents\My SAS Files\9.4\gapminder.sas

import glob
path = 'C:\\'
files = (f for f in glob.glob(path + '**/Read*.*', recursive=True))
for f in files:
    print(f)

Count the number of files in a folder

import os
cpt = sum([len(files) for r, d, files in os.walk(r'C:\SASHELP_Raw1')])
cpt

184

Move .SAS files from one folder to another folder

import os
import shutil
sourcepath=r'C:\NIH'
sourcefiles = os.listdir(sourcepath)
destinationpath = r'C:\Data'
for file in sourcefiles:
    if file.endswith('.sas'):
        shutil.move(os.path.join(sourcepath,file), os.path.join(destinationpath,file))

Copy .SAS files from one folder to another folder

Unicode Error "unicodeescape" ... cannot pen text file

You either need to duplicate all backslashes:

"C:\\Users\\Eric\\Desktop\\beeline.txt"
Or prefix the string with r (to produce a raw string):

r"C:\Users\Eric\Desktop\beeline.txt"

import glob, os, shutil

files = glob.iglob(os.path.join(r'C:\Data', "*.sas"))
for file in files:
    if os.path.isfile(file):
        shutil.copy2(file, r'C:\NIH')

import glob
path = 'C:\\'
files = (f for f in glob.glob(path + '**/MEPS_zip_links*.xlsx', recursive=True))
for f in files:
    print(f)

from pathlib import Path
dir =  Path(r'C:\NIH')
files = dir.glob('*.sas')
for i in files:
    print(i)

from pathlib import Path
dir =  Path(r'C:\SASCourse')
files = dir.glob('*.sas')
for i in files:
    print(i)

from saspy import autocfg
autocfg.main()

CFGFILE ALREADY EXISTS: C:\Users\pmuhuri\AppData\Local\Continuum\anaconda3\lib\site-packages\saspy\sascfg_personal.py

fd = open(r'C:\Explore\SAS\Lesson14\SAS_Codes\Ex_sum_statement_cars_data.sas')
print(fd.read())
fd.close()

proc sort data = sashelp.cars; by make; run;
data cars;
  set sashelp.cars;
  count + 1;
  by make;
  if first.make then count = 1;
  if last.make;
run;
proc print data=cars;
var make count;
run;
proc sort data=sashelp.cars out=cars; by make; run;
data counts (keep= make model type count_of_make);
   set cars; by make;
   if first.make then count_of_make=0;
    count_of_make+1;
   if last.make;
  run;
proc print data=&syslast; run;

How to know the pandas version

import pandas as pd
pd.show_versions()

from platform import python_version
print(python_version())

3.7.4

import sys
sys.path

import sys
#print(sys.path)
for path in sys.path: print(path)

pprint : Data pretty printer in Python The following code also works.

import sys
import pprint
# pretty print module search paths
pprint.pprint(sys.path)

pathlib — Object-oriented filesystem paths

import pathlib
py = pathlib.Path().glob("*.ipynb")
for file in py: 
    print(file)

Diagnostics2_sascfg_personal.ipynb
Python_Pandas_ML_refs.ipynb
Week14_Cheatsheet_Python_R.ipynb
Week14_Jupyter_Part3.ipynb
Week14_Pandas_Part4.ipynb
Week14_Pandas_Part5.ipynb
Week14_Python_Part2.ipynb
Week14_SASPy_Part1.ipynb

# change the current directory to specified directory 
os.chdir(r"C:\Data") 
print("Directory changed") 

Directory changed

%pwd

'C:\\Data'