Prepare file path using Java.IO

Question

I am very much new to scala and prepared below function.In below function,i need to check if particular path exists:

def insertData(df: DataFrame, path: String): Unit = {
    import sc.sqlContext.implicits._

    // get output path to export the parquet files
    val MPath = path+"/output"
    var getDates = df.select("dates").distinct().collect().map(_(0)).toList
    var invalidDates = new ListBuffer[String]()

    // check if output path is present or not

    if (new JFile(MPath ).exists) {

      for (dates <- getDates) {
        if (new JFile(MPath +"/Date=" + dates).exists) {


          invalidDates += "Date=" + dates.toString

          FileUtils.deleteDirectory(new JFile(MPath+ "/Date=" + dates))
        }

        else
        {
          log.info(s"No parquet associated with Date")
        }

      }
    }

    else
    {

      new JFile(MPath).mkdirs()
    }

     }

Here ,I am using new JFile(MPath ) and then in for loop doing concatenation using new JFile(MPath +"/Date=" + dates).exists) .I just need to avoid re-writing new JFile .Is there any method to avoid this

Answer 1

Use Hadoop's FileSystem:

def doesPathExist(pathStr: String):Boolean = {
   import org.apache.hadoop.conf.Configuration
   import org.apache.hadoop.fs.{FileSystem, Path}

   val path = new Path(pathStr)
   val fileSystem = path.getFileSystem(new Configuration)

   fileSystem.exists(path)
}