在eclipse中编写HDFS的Java程序

1.开始我们需要配置jdk+eclipse+maven,文章参考链接https://blog.csdn.net/qq_44240521/article/details/90574089 2.在Eclipse里新建Maven Project 【file】-【new】-【Maven project】 如下图自动生成Maven项目，新建包hdfs.files，包里新建class：4个java文件。 3.编写程序 在pom.xml里添加

<dependencies> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-hdfs</artifactId> <version>2.7.3</version> </dependency> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-client</artifactId> <version>2.7.3</version> </dependency> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-common</artifactId> <version>2.7.3</version> </dependency> </dependencies>

在HDFSDownload.java（下载）里编写(hdfs上需要有word.txt)

package hdfs.files; import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; public class HDFSDownload { private static InputStream input; private static OutputStream output; public static void main(String[] args) throws IOException{ System.setProperty("HADOOP_USER_NAME", "root"); Configuration conf=new Configuration(); conf.set("fs.defaultFS", "hdfs://centos01:9000"); FileSystem client=FileSystem.get(conf); output = new FileOutputStream("e:\\eclipse\\word.txt"); input =client.open(new Path("/word.txt")); byte[] buffer=new byte[1024]; int len=0; while((len=input.read(buffer))!=-1) { output.write(buffer,0,len); } output.flush(); input.close(); output.close(); } }

在HDFSFilelfExist.java（判断hdfs文件是否存在）里编写

package hdfs.files; import java.io.IOException; import org.apache.hadoop. conf.Configuration; import org.apache.hadoop. fs.FileSystem; import org.apache.hadoop.fs.Path; public class HDFSFilelfExist{ public static void main(String[] args) throws IOException{ System. setProperty( "HADOOP_ USER_ NAME", "root"); Configuration conf=new Configuration(); conf . set("fs.defaultFS", "hdfs://centos01:9000"); FileSystem client=FileSystem. get(conf); String fileName="/aadir"; if(client.exists(new Path(fileName))) { System.out.println("文件存在! "); } else { System.out.println(" 文件不存在! "); } } }

在HDFSMKdir.java（创建目录）里编写

package hdfs.files; import java.io.IOException; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; public class HDFSMKdir{ public static void main(String[] args) throws IOException { System.setProperty( "HADOOP_ USER_ NAME", "root"); Configuration conf=new Configuration(); conf.set("fs.defaultFS", "hdfs://centos01:9000"); FileSystem client=FileSystem.get(conf); client.mkdirs( new Path("/aadir")); client.close(); System.out.println(" successfully!"); } }

在HDFSUpload .java（上传本地文件到hdfs）里编写

package hdfs.files; import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; public class HDFSUpload { private static InputStream input; private static OutputStream output; public static void main(String[] args) throws IOException{ Configuration conf=new Configuration(); conf.set("fs.defaultFS", "hdfs://centos01:9000"); FileSystem client=FileSystem.get(conf); input = new FileInputStream("e:\\eclipse\\test.txt"); output =client.create(new Path("/aadir/test.txt")); byte[] buffer=new byte[1024]; int len=0; while((len=input.read(buffer))!=-1) { output.write(buffer,0,len); } output.flush(); input.close(); output.close(); } }

4.运行程序 运行下载 运行判断文件是否存在 好，就演示到这 其中可能要用到winutils，下载地址https://github.com/rucyang/hadoop.dll-and-winutils.exe-for-hadoop2.7.3-on-windows_X64/tree/master，然后配置环境变量