#!/usr/bin/perl -w
use utf8;
# use to validate:
# python ~/projects/bs/GIT/cvrfparse/cvrfparse.py -C ~/projects/bs/GIT/cvrfparse/schemata/catalog.xml -S ~/projects/bs/GIT//cvrfparse/schemata/cvrf/1.1/cvrf.xsd -f ~/public_html/cvrf/cvrf-SUSE-Storage-4-2017-56.xml -V
#
# For CVRF 1.2:
# https://github.com/oasis-open/csaf-parser
# run something like:
#      python3 cvrf_util.py --file /mounts/mirror/SuSE/ftp.suse.com/pub/projects/security/cvrf1.2/cvrf-opensuse-su-2021\:0639-1.xml --schema schemata/cvrf/1.2/cvrf.xsd --cvrf-version 1.2 --output-format txt --prod Branch
#
#
use strict;
my $dn = `dirname $0`;chomp($dn);
my $pwd = `pwd`;chomp($pwd);
if ($dn !~ /^\//) { $dn = $pwd . "/" . $dn; }
push @INC,$dn;
my $cverepobase=`dirname $dn`;
chomp($cverepobase);

use POSIX qw/strftime setlocale LC_TIME/;
use List::MoreUtils qw/uniq/;
use IO::File;
use XML::Writer;
use DateTime;
require SMASHData;
require CanDBReader;
require UpdateInfoReader;
UpdateInfoReader->import_product_updates();
UpdateInfoReader->import_images();
require CVEListReader;

my $mode = pop @ARGV;

if (!defined($mode)) { $mode = "fast"; }

my $defserial = 1;

my %mypackages = ();
my %myversions = ();
my %mytests = ();
my %myplatforms = ();

# get the severity
foreach my $id (sort keys %SMASHData::severity) {
	next if ($id =~ /CVE-/);
	if ($id =~ /bnc#(\d*)/) {
		my $nr = $1;
		foreach my $cve (sort keys %CanDBReader::bugzillas) {
			if (grep(/$nr/,$CanDBReader::bugzillas{$cve})) {
				$SMASHData::severity{$cve} = $SMASHData::severity{$id};
				delete $SMASHData::severity{$id};
				print STDERR "severity merger: rating $SMASHData::severity{$cve} merged $id into $cve\n";
				last;
			}
		}
	}
}

sub
filter_score($) {
	my ($score) = @_;

	$score =~ s/^(\d*\.\d)\d*$/$1/;
	return $score;
}

sub generate_cvrf() {
	# products for which we want to dump oval
	my @products = ();
	my %xx;

	my %allpatches;

	my ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) = localtime();

	$year += 1900;


	# collect all patches we want to do by product (SLED/SLES, various SPs)

	foreach my $product (keys %UpdateInfoReader::patches) {
		my %patches = %{$UpdateInfoReader::patches{$product}};

		next if ($product =~ /SUSE:SLE-/);
		next if ($product =~ /openSUSE 1/);
		next if ($product =~ /openSUSE Ever/);

		push @products, $product;

		print STDERR "scanning patches for $product\n" if (-t STDERR);

		foreach my $patch (keys %patches) {
			if ($UpdateInfoReader::patchinqa{$patch}) {
				print STDERR "$patch is in QA\n" if (-t STDERR);
				next;
			}
			$allpatches{$patch} = 1;
		}
	}

	my @notices = ();

	if ($mode eq "all") {
		@notices = keys %CanDBReader::susenotice2patches;
	} else {
		@notices = grep (/SU-$year/, keys %CanDBReader::susenotice2patches);
	}

	foreach my $notice (sort @notices) {
		my @patches = sort keys %{$CanDBReader::susenotice2patches{$notice}};

		my $patch;
		my $codestreampatch;

		foreach my $xpatch (@patches) {
			if (($xpatch =~ /^SUSE-\d*-\d*$/)  && (!defined($codestreampatch))) { # codestream
				$codestreampatch = $xpatch;
			}
                        # skip removed patches. this has happened as we released patches during development and later deleted them as they were included in GA.
                        $patch = $xpatch if (defined($UpdateInfoReader::patchtype{$xpatch}));

		}
		if (!defined($patch)) {
			print STDERR "ERROR: notice $notice has no valid patches out of: " . join(",",@patches) . "\n";
			next;
		}

		# codestreampatch is only on SLE, but here we process also openSUSE
		if (defined($codestreampatch) && ($codestreampatch =~ /SUSE(-\d*-\d*)$/)) {
			my $patchid = $1;
			# add all images and containers here ... wild hack
			push @patches, grep (/^Image.*$patchid$/,keys %allpatches);
			push @patches, grep (/^Container.*$patchid$/,keys %allpatches);
		}

		unless (defined($UpdateInfoReader::patchtype{$patch})) {
			print STDERR "SKIP $notice as $patch is not known.\n";
			print STDERR "patches: " . join(",",@patches) . "\n";
			next;
		}
		if ($UpdateInfoReader::patchtype{$patch} ne "security") {
			# check if we have CVEs in references.
			unless (grep (/CVE/,keys %{$UpdateInfoReader::patchreferences{$patch}})) {
				print STDERR "SKIP $notice as $patch is not security or has no CVEs.\n";
				print STDERR "patches: " . join(",",@patches) . "\n";
				next;
			}
		}

		@patches = sort @patches;

		print STDERR "$notice ...\n" if (-t STDERR);
		if (!defined($UpdateInfoReader::patchpackages{$patch})) {
			print STDERR "SKIP $notice: deleted patch $patch?\n";
			next;
		}

		#print STDERR "starting affected products... " . @products . "\n" if -t STDERR;
		# for which products do we have this patch
		my %affectedproducts = ();
		foreach my $xpatch (@patches) {
			foreach my $product (@products) {
				next unless (defined($UpdateInfoReader::patches{$product}->{$xpatch}));
				$affectedproducts{$product} = 1;
			}
		}
		my @affectedproducts = sort keys %affectedproducts;
		#print STDERR "...finished affected products.\n" if -t STDERR;
		# emit for both CVRF 1.1 and 1.2...
		foreach my $cvrfver ("1.1", "1.2") {

			my $xpath = "/mounts/mirror/SuSE/ftp.suse.com/pub/projects/security/cvrf";
			if ($cvrfver eq "1.2") {
				$xpath .= "1.2";
			}
			$xpath .= "/";
			chdir($xpath)||die "chdir $xpath:$!";

			#print "scanning patch $patch\n";

			my $fn = "\Lcvrf-$notice.xml";

			# make faster, generate only once.
			next if (-f $fn && ($mode ne "all"));

			#print STDERR "start generating $cvrfver file\n" if -t STDERR;
			my $output = IO::File->new(">$fn.new") || die "opening $fn.new:$!";
			my $writer = XML::Writer->new(OUTPUT => $output, NEWLINES => 1, ENCODING => "utf-8");
			$writer->xmlDecl("UTF-8");
			if ($cvrfver eq "1.1") {
				$writer->startTag("cvrfdoc",
					"xmlns" => "http://www.icasi.org/CVRF/schema/cvrf/1.1",
					"xmlns:cvrf" => "http://www.icasi.org/CVRF/schema/cvrf/1.1",
				);
			} else {
				$writer->startTag("cvrfdoc",
					"xmlns:xsd" => "http://www.w3.org/2001/XMLSchema",
					"xmlns:cpe" => "http://cpe.mitre.org/language/2.0",
					"xmlns:cvrf" => "http://docs.oasis-open.org/csaf/ns/csaf-cvrf/v1.2/cvrf",
					"xmlns:cvrf-common" => "http://docs.oasis-open.org/csaf/ns/csaf-cvrf/v1.2/common",
					"xmlns:cvssv2" => "http://scap.nist.gov/schema/cvss-v2/1.0",
					"xmlns:cvssv3" => "https://www.first.org/cvss/cvss-v3.0.xsd",
					"xmlns:dc" => "http://purl.org/dc/elements/1.1/",
					"xmlns:ns0" => "http://purl.org/dc/elements/1.1/",
					"xmlns:prod" => "http://docs.oasis-open.org/csaf/ns/csaf-cvrf/v1.2/prod",
					"xmlns:scap-core" => "http://scap.nist.gov/schema/scap-core/1.0",
					"xmlns:sch" => "http://purl.oclc.org/dsdl/schematron",
					"xmlns:vuln" => "http://docs.oasis-open.org/csaf/ns/csaf-cvrf/v1.2/vuln",
					"xmlns:xsi" => "http://www.w3.org/2001/XMLSchema-instance",
					"xmlns" => "http://docs.oasis-open.org/csaf/ns/csaf-cvrf/v1.2/cvrf",
				);
			}

			$writer->dataElement("DocumentTitle", $UpdateInfoReader::patchtitle{$patch}, "xml:lang" => "en");
			$writer->dataElement("DocumentType", "SUSE Patch");

			$writer->startTag("DocumentPublisher", "Type" => "Vendor");
				$writer->dataElement("ContactDetails", "security\@suse.de");
				$writer->dataElement("IssuingAuthority", "SUSE Security Team");
			$writer->endTag("DocumentPublisher");

			$writer->startTag("DocumentTracking");

				$writer->startTag("Identification");
					$writer->dataElement("ID", $notice);
				$writer->endTag("Identification");

				$writer->dataElement("Status", "Final");
				$writer->dataElement("Version", "1");

				if (!defined($UpdateInfoReader::patchissued{$patch})) {
					die "$patch has no issues?\n";
				}
				my $dt = DateTime->from_epoch (epoch  => $UpdateInfoReader::patchissued{$patch});

				$writer->startTag("RevisionHistory");
					$writer->startTag("Revision");
						$writer->dataElement("Number", "1");
						$writer->dataElement("Date", $dt->iso8601()."Z");
						$writer->dataElement("Description", "current");
					$writer->endTag("Revision");
				$writer->endTag("RevisionHistory");

				$writer->dataElement("InitialReleaseDate",$dt->iso8601()."Z");
				$writer->dataElement("CurrentReleaseDate",$dt->iso8601()."Z");

				$writer->startTag("Generator");
					$writer->dataElement("Engine","cve-database/bin/generate-cvrf.pl");
					$writer->dataElement("Date","2017-02-24T01:00:00Z");
				$writer->endTag("Generator");

			$writer->endTag("DocumentTracking");


			$writer->startTag("DocumentNotes");

			$writer->dataElement(
				"Note",
				$UpdateInfoReader::patchtitle{$patch},
				"Title" => "Topic", "Type" => "Summary", "Ordinal" => "1", "xml:lang" => "en"
				);
			$writer->dataElement(
				"Note",
				$UpdateInfoReader::patchdescription{$patch},
				"Title" => "Details", "Type" => "General", "Ordinal" => "2", "xml:lang" => "en"
				);
			$writer->dataElement(
				"Note",
				"The CVRF data is provided by SUSE under the Creative Commons License 4.0 with Attribution (CC-BY-4.0).",
				"Title" => "Terms of Use", "Type" => "Legal Disclaimer", "Ordinal" => "3", "xml:lang" => "en"
			);

			$writer->dataElement(
				"Note",
				join(",",@patches),
				"Title" => "Patchnames", "Type" => "Details", "Ordinal" => "4", "xml:lang" => "en"
				);
			$writer->endTag("DocumentNotes");

			$writer->dataElement(
				"DocumentDistribution",
				"Copyright SUSE LLC under the Creative Commons License 4.0 with Attribution (CC-BY-4.0)",
				"xml:lang" => "en"
			);

			$writer->startTag("DocumentReferences");

				# translate suse note
				# SUSE-SU-2017:0367-1
				#https://www.suse.com/support/update/announcement/2017/suse-su-20170367-1/

				my $urlnotice = "\L$notice";
				$urlnotice =~ /su-(\d*):/;
				my $noticeyear = $1;
				$urlnotice =~ s/://;
				$urlnotice = "https://www.suse.com/support/update/announcement/$noticeyear/$urlnotice/";

				if ($notice =~ /^SUSE/) {	# only SUSE notices are uploaded to suse.com/support/update
					$writer->startTag("Reference", "Type" => "Self");
						$writer->dataElement("URL", $urlnotice);
						$writer->dataElement("Description", "Link for $notice");
					$writer->endTag("Reference");
				} else {
					$urlnotice = $CanDBReader::advisoryid2url{$notice};
				}

				if (defined($CanDBReader::advisoryid2url{$notice})) {
					$writer->startTag("Reference", "Type" => "Self");
						$writer->dataElement("URL", $CanDBReader::advisoryid2url{$notice});
						$writer->dataElement("Description", "E-Mail link for $notice");
					$writer->endTag("Reference");
				}
				$writer->startTag("Reference", "Type" => "Self");
					$writer->dataElement("URL", "https://www.suse.com/support/security/rating/");
					$writer->dataElement("Description", "SUSE Security Ratings");
				$writer->endTag("Reference");
				my %references = %{$UpdateInfoReader::patchreferences{$patch}};
				foreach my $reference (sort keys %references) {
					if ($reference =~ /CVE/) {
						$writer->startTag("Reference", "Type" => "Self");
							$writer->dataElement("URL", "https://www.suse.com/security/cve/$reference/");
							$writer->dataElement("Description", "SUSE CVE $reference page");
						$writer->endTag("Reference");
					} else {	#assume bug
						$writer->startTag("Reference", "Type" => "Self");
							$writer->dataElement("URL", "https://bugzilla.suse.com/$reference");
							$writer->dataElement("Description", "SUSE Bug $reference");
						$writer->endTag("Reference");
					}
				}
			$writer->endTag("DocumentReferences");

			if ($cvrfver eq "1.1") {
				$writer->startTag("ProductTree", "xmlns" => "http://www.icasi.org/CVRF/schema/prod/1.1");
			} else {
				$writer->startTag("ProductTree", "xmlns" => "http://docs.oasis-open.org/csaf/ns/csaf-cvrf/v1.2/prod");
			}

			# list of products fixed by the patches of this notice.
			foreach my $product (@affectedproducts) {
				$writer->startTag("Branch", "Type" => "Product Family", "Name" => $product);
					$writer->startTag("Branch", "Type" => "Product Name", "Name" => $product);
						my @dataelement = (
							"FullProductName",      $product,
							"ProductID" => $product,
						);
						if (&SMASHData::get_cpe($product)) {
							push @dataelement,'CPE',&SMASHData::get_cpe($product);
						} else {
							warn "no cpe found for $product\n";
						}
						$writer->dataElement(@dataelement);
					$writer->endTag("Branch");
				$writer->endTag("Branch");
			}

			my %havepackagever = ();

			foreach my $xpatch (@patches) {
				if (!defined($UpdateInfoReader::patchpackages{$xpatch})) {
					print STDERR "no patch packages for $xpatch?\n";
					next;
				}
				my %packages = %{$UpdateInfoReader::patchpackages{$xpatch}};
				foreach my $package (sort keys %packages) {
					next if ($package =~ /-(debuginfo|debugsource)/);

					my $pkgver = "$package-$packages{$package}";

					if (!defined($havepackagever{$pkgver})) {
						$havepackagever{$pkgver} = 1;

						$writer->startTag("Branch", "Type" => "Product Version", "Name" => $pkgver);
							$writer->dataElement( "FullProductName", $pkgver, "ProductID" => $pkgver);
						$writer->endTag("Branch");
					}
				}
			}

			my %product_packages = ();
			# Foreach patch in the notice, dump the packages and the product:packages relation
			foreach my $product (@affectedproducts) {
				foreach my $xpatch (@patches) {
					next unless (defined($UpdateInfoReader::patches{$product}->{$xpatch}));
					next unless (defined($UpdateInfoReader::patchpackages{$xpatch}));

					# only print the ones for this product.
					my %packages = %{$UpdateInfoReader::patchpackages{$xpatch}};
					foreach my $package (sort keys %packages) {
						next if ($package =~ /-(debuginfo|debugsource)/);

						my $pkgver = "$package-$packages{$package}";

						my $prodpkgid = "$product:$pkgver";
						$product_packages{$prodpkgid} = 1;
						# push @product_package,$prodpkgid;
						$writer->startTag(
							"Relationship",
							"ProductReference" => $pkgver,
							"RelationType" => "Default Component Of",
							"RelatesToProductReference" => $product
						);
							$writer->dataElement(
								"FullProductName",
								"$pkgver as a component of $product",
								"ProductID" => $prodpkgid
							);
						$writer->endTag("Relationship");
					}
				}
			}

			$writer->endTag("ProductTree");

			my @product_package = sort keys %product_packages;

			#print STDERR "finished product tree\n" if -t STDERR;

			# assumed to be the same for all patches of the same notice
			%references = %{$UpdateInfoReader::patchreferences{$patch}};
			my $ordinal = 1;
			foreach my $reference (sort keys %references) {
				next unless ($reference =~ /^CVE-/);
				if ($cvrfver eq "1.1") {
					$writer->startTag("Vulnerability", "Ordinal" => $ordinal, "xmlns" => "http://www.icasi.org/CVRF/schema/vuln/1.1");
				} else {
					$writer->startTag("vuln:Vulnerability", "Ordinal" => $ordinal, "xmlns" => "http://docs.oasis-open.org/csaf/ns/csaf-cvrf/v1.2/vuln");
					# Tags missing: Title, ID
				}
					$writer->startTag("Notes");
						$writer->dataElement(
							"Note",
							get_description($reference) || "unknown",
							"Title" => "Vulnerability Description",
							"Type" => "General",
							"Ordinal" => 1, 	# FIXME once we have more than one note
							"xml:lang" => "en"
						);
					$writer->endTag("Notes");
					# DiscoveryDate
					# ReleaseDate
					$writer->dataElement("CVE",$reference);

					$writer->startTag("ProductStatuses");
						$writer->startTag("Status","Type" => "Fixed");
							foreach my $package (@product_package) {
								$writer->dataElement("ProductID",$package);
							}
						$writer->endTag("Status","Type" => "Fixed");
					$writer->endTag("ProductStatuses");


					$writer->startTag("Threats");
						$writer->startTag("Threat", "Type" => "Impact");
							my $severity = $SMASHData::severity{$reference};
							if (!defined($severity)) {
								$severity = $UpdateInfoReader::patchseverity{$patch};
							}
							if (!defined($severity)) {
								print STDERR "$notice / $patch - no severity?\n" if (-t STDERR);
								$severity = "moderate";
							}
							$writer->dataElement("Description", $severity);
						$writer->endTag("Threat");
					$writer->endTag("Threats");

					if ($cvrfver eq "1.1") {
						my $cvss = &SMASHData::get_cvss_issue($reference);
						if (defined($cvss)) {
							if (defined($cvss->{'SUSE'}) || defined($cvss->{'National Vulnerability Database'})) {
								$writer->startTag("CVSSScoreSets");
									if (defined($cvss->{'SUSE'})) {
										$writer->startTag("ScoreSet");
											$writer->dataElement("BaseScore", filter_score($cvss->{'SUSE'}->{base_score}));
											$writer->dataElement("Vector", $cvss->{'SUSE'}->{base_vector});
										$writer->endTag("ScoreSet");
									}
									if (defined($cvss->{'National Vulnerability Database'})) {
										$writer->startTag("ScoreSet");
											$writer->dataElement("BaseScore", filter_score($cvss->{'National Vulnerability Database'}->{base_score}));
											$writer->dataElement("Vector", $cvss->{'National Vulnerability Database'}->{base_vector});
										$writer->endTag("ScoreSet");
									}

								$writer->endTag("CVSSScoreSets");
							}
						}
					} else {	# 1.2
						my $cvss = &SMASHData::get_cvss_issue($reference);
						my $cvss3 = &SMASHData::get_cvssv3_issue($reference);
						if (defined($cvss) || defined($cvss3)) {
							if (	defined($cvss->{'SUSE'}) || defined($cvss->{'National Vulnerability Database'}) ||
								defined($cvss3->{'SUSE'}) || defined($cvss3->{'National Vulnerability Database'})
							) {
								$writer->startTag("CVSSScoreSets");
									# emit either SUSE or NVD cvss v2
									if (defined($cvss->{'SUSE'})) {
										$writer->startTag("ScoreSetV2");
											$writer->dataElement("BaseScoreV2", filter_score($cvss->{'SUSE'}->{base_score}));
											$writer->dataElement("VectorV2", $cvss->{'SUSE'}->{base_vector});
										$writer->endTag("ScoreSetV2");
									} else  {
										if (defined($cvss->{'National Vulnerability Database'})) {
											$writer->startTag("ScoreSetV2");
												$writer->dataElement("BaseScoreV2", filter_score($cvss->{'National Vulnerability Database'}->{base_score}));
												$writer->dataElement("VectorV2", $cvss->{'National Vulnerability Database'}->{base_vector});
											$writer->endTag("ScoreSetV2");
										}
									}
									if (defined($cvss3->{'SUSE'})) {
										$writer->startTag("ScoreSetV3");
											$writer->dataElement("BaseScoreV3", filter_score($cvss3->{'SUSE'}->{base_score}));
											$writer->dataElement("VectorV3", $cvss3->{'SUSE'}->{base_vector});
										$writer->endTag("ScoreSetV3");
									} else {
										if (defined($cvss3->{'National Vulnerability Database'})) {
											$writer->startTag("ScoreSetV3");
												$writer->dataElement("BaseScoreV3", filter_score($cvss3->{'National Vulnerability Database'}->{base_score}));
												$writer->dataElement("VectorV3", $cvss3->{'National Vulnerability Database'}->{base_vector});
											$writer->endTag("ScoreSetV3");
										}
									}

								$writer->endTag("CVSSScoreSets");
							}
						}
					}
					$writer->startTag("Remediations");
						$writer->startTag("Remediation", "Type" => "Vendor Fix");
							# could link to the advisory? or cve page?
							$writer->dataElement(
								"Description",
	"To install this SUSE Security Update use the SUSE recommended installation methods like YaST online_update or \"zypper patch\".\n",
								"xml:lang" => "en"
							);
							$writer->dataElement( "URL", $urlnotice);
						$writer->endTag("Remediation");
					$writer->endTag("Remediations");

					$writer->startTag("References");
						$writer->startTag("Reference");
							$writer->dataElement("URL", "https://www.suse.com/security/cve/$reference.html");
							$writer->dataElement("Description", $reference);
						$writer->endTag("Reference");

						if (defined($CanDBReader::bugzillas{$reference})) {
							my %bugs = map { $_ => 1 } split (/,/,$CanDBReader::bugzillas{$reference});
							foreach my $bug (sort keys %bugs) {
								$writer->startTag("Reference");
									$writer->dataElement("URL", "https://bugzilla.suse.com/$bug");
									$writer->dataElement("Description", "SUSE Bug $bug");
								$writer->endTag("Reference");
							}
						} else {
							warn "no bugzillas for $reference?\n";
						}
					$writer->endTag("References");

				if ($cvrfver eq "1.1") {
					$writer->endTag("Vulnerability");
				} else {
					$writer->endTag("vuln:Vulnerability");
				}
				$ordinal++;
			}

			$writer->endTag("cvrfdoc");
			$writer->end();
			$output->close();

			#print STDERR "finish generating $cvrfver file, running xmllint\n" if -t STDERR;
			die ">$fn.new is 0 bytes, disk full?" if (! -s "$fn.new");
			system("xmllint --format $fn.new >$fn.new.formatted");
			my $oldcvrf = "";
			my $newcvrf;

			if (! -s "$fn.new.formatted") {
				warn ">$fn.new.formatted is 0 bytes, conversion failed or disk full?";
				unlink("$fn.new");
				unlink("$fn.new.formatted");
				next;
			}

			if (open(CVRF,"<$fn.new.formatted")) {
				$newcvrf = join("",<CVRF>);
				close(CVRF);
			}
			if (open(CVRF,"<$fn")) {
				$oldcvrf = join("",<CVRF>);
				close(CVRF);
			}
			if ($oldcvrf ne $newcvrf) {
				if (system("diff -uN $fn $fn.new.formatted")) {
					rename("$fn.new.formatted",$fn);
					unlink("$fn.new");
				} else {
					print STDERR "diff for $fn not detected by reader?\n";
					unlink("$fn.new");
					unlink("$fn.new.formatted");
				}
			} else {
				unlink("$fn.new");
				unlink("$fn.new.formatted");
			}
			print STDERR "done generating $cvrfver file\n" if -t STDERR;
		}
	}
}

umask 022;

generate_cvrf();

print STDERR "Done creating ... packing tarballs.\n" if -t STDERR;

chdir("/mounts/mirror/SuSE/ftp.suse.com/pub/projects/security/");
system("tar cjf cvrf.tar.bz2 cvrf/");
system("tar cjf cvrf1.2.tar.bz2 cvrf1.2/");


print "SUCCESS\n";