Writing dataframe to blob with AzurePutbBlob about azuresmr

[rsmith54]

I don't like it, but I used the following function (from http://r.789695.n4.nabble.com/write-csv-to-text-string-td4704560.html) to write a temporary file and then uploaded that using the AzureSMR API.

write.csv.str <- function(df, row.names = F) {
    filepath <- tempfile()
    f <- file(filepath, "w")
    write.csv(df, f, row.names = row.names)
    close(f)
    s <- readr::read_file(filepath)
    s
}

[atroiano]

@rsmith54 This is the method I am using, thanks for providing.

I would suggest you use fwrite from the data.table package instead of write.csv. For text columns, write.csv won't delimit them correctly so when you read the blob, it will error.

[krunalhd]

Have you been able to achieve this successfully? If yes can you share your code? For me it's failing and I pasted my issue here as well.

[ramnov]

Maybe you can try this (writing iris dataframe to blob) :

write.csv(iris, textConnection("irisString", "w"), row.names = F)
azurePutBlob(sc, storageAccount = "testmystorage1", container = "opendata", 
             contents = paste(irisString, collapse = '\n'),
             blob = "HELLO") 

[atroiano]

For large DFs, I created a function that will load the data into large chunks and read the loaded data back into a DF

save_chunks<-function(df,tenant_id,client_id,auth_key,auth_type,resource_group,storage_account,container_name,blob_folder,blob_name){
  write.csv.str <- function(df, row.names = F) {
    filepath <- tempfile()
    fwrite(df, filepath, row.names = row.names)
    s <- readr::read_file(filepath)
    s
  }
  end <- 0
  id <- 1
  start <- 1
  num_rows<-round(64000000/((object.size(df)[1])/nrow(df)),0)  
  end <- start+num_rows
  sc <- createAzureContext(tenantID =tenant_id
                           #application id = clientID
                           ,clientID = client_id
                           #the password 
                           ,authKey= auth_key
                           ,authType = auth_type)
  
  azureSAGetKey(sc,resourceGroup = resource_group, storageAccount = storage_account )
  while (end <= nrow(df) ) {
    #get size of one row and figure out how many rows per chunk, limiting to 64 MB
    #id will save multiple files
    chunk<-df[start:end,]
    
    azurePutBlob(sc
                 ,storageAccount = storage_account
                 ,container = container_name
                 ,contents = write.csv.str(chunk)
                 ,blob = paste(blob_folder,'/',blob_name,'_',id,sep='')) 
    if (end ==  nrow(df) )
      return(NULL)
    id<- id+1
    start <- end+1
    end <- start+num_rows

    if (end >= nrow(df))
      end <- nrow(df)
    print(start)
    print(end)
  }
}
get_blob_data <- function(tenant_id,client_id,auth_key,auth_type,resource_group,storage_account,container_name,blob_folder,blob_name) {
  sc <- createAzureContext(tenantID =tenant_id
                           #application id = clientID
                           ,clientID = client_id
                           #the password 
                           ,authKey= auth_key
                           ,authType = auth_type)
  
  azureSAGetKey(sc,resourceGroup = resource_group, storageAccount = storage_account )
  
  all_files<-azureBlobLS(sc,directory = blob_folder,recursive = F,storageAccount = storage_account,resourceGroup = resource_group,container = container_name)
  total_files <- nrow(all_files)
  print(total_files)
  start <- 1
  while (start <=total_files){
    print (start)
    data<-azureGetBlob(azureActiveContext = sc,storageAccount = storage_account, directory = blob_folder,container = container_name,resourceGroup = resource_group,blob=paste(blob_name,'_',start,sep=''))
    filepath<-tempfile()
    write_lines(data,filepath)
    if (!exists('blob_df')){
      blob_df<-fread(filepath,showProgress = F)
      start <- start+1
    } else {
      blob_df %<>% rbind(.,fread(filepath, showProgress = F))
      start<-start+1
    }
  }
  return(blob_df)
}