Merge branch 'frappe:develop' into custom-doctype-webform-perm

.github/helper/roulette.py

@@ -25,7 +25,7 @@ def is_ci(file):
 	return ".github" in file
 
 def is_frontend_code(file):
-	return file.endswith((".css", ".scss", ".less", ".sass", ".styl", ".js", ".ts"))
+	return file.lower().endswith((".css", ".scss", ".less", ".sass", ".styl", ".js", ".ts", ".vue"))
 
 def is_docs(file):
 	regex = re.compile(r'\.(md|png|jpg|jpeg|csv)$|^.github|LICENSE')
@@ -38,8 +38,12 @@ def is_docs(file):
 	pr_number = os.environ.get("PR_NUMBER")
 	repo = os.environ.get("REPO_NAME")
 
-	if not files_list and pr_number:
-		files_list = get_files_list(pr_number=pr_number, repo=repo)
+	# this is a push build, run all builds
+	if not pr_number:
+		os.system('echo "::set-output name=build::strawberry"')
+		sys.exit(0)
+
+	files_list = files_list or get_files_list(pr_number=pr_number, repo=repo)
 
 	if not files_list:
 		print("No files' changes detected. Build is shutting")

.github/workflows/patch-mariadb-tests.yml

@@ -2,6 +2,11 @@ name: Patch
 
 on: [pull_request, workflow_dispatch]
 
+
+concurrency:
+  group: patch-mariadb-develop-${{ github.event.number }}
+  cancel-in-progress: true
+
 jobs:
   test:
     runs-on: ubuntu-18.04

.github/workflows/server-mariadb-tests.yml

@@ -6,6 +6,11 @@ on:
   push:
     branches: [ develop ]
 
+concurrency:
+  group: server-mariadb-develop-${{ github.event.number }}
+  cancel-in-progress: true
+
+
 jobs:
   test:
     runs-on: ubuntu-18.04
@@ -131,17 +136,29 @@ jobs:
           COVERALLS_SERVICE_NAME: ${{ github.event_name == 'pull_request' && 'github' || 'github-actions' }}
           COVERALLS_PARALLEL: true
 
+      - run: echo ${{ steps.check-build.outputs.build }} > guess-the-fruit.txt
+      - uses: actions/upload-artifact@v1
+        with:
+          name: fruit
+          path: guess-the-fruit.txt
+
   coveralls:
     name: Coverage Wrap Up
     needs: test
-    if: ${{ needs.test.steps.check-build.build == 'strawberry' }}
     container: python:3-slim
     runs-on: ubuntu-18.04
     steps:
+      - uses: actions/download-artifact@v1
+        with:
+          name: fruit
+      - run: echo "WILDCARD=$(cat fruit/guess-the-fruit.txt)" >> $GITHUB_ENV
+
       - name: Clone
+        if: ${{ env.WILDCARD == 'strawberry' }}
         uses: actions/checkout@v2
 
       - name: Coveralls Finished
+        if: ${{ env.WILDCARD == 'strawberry' }}
         run: |
           cd ${GITHUB_WORKSPACE}
           pip3 install coverage==5.5

.github/workflows/server-postgres-tests.yml

@@ -4,6 +4,10 @@ on:
   pull_request:
   workflow_dispatch:
 
+concurrency:
+  group: server-postgres-develop-${{ github.event.number }}
+  cancel-in-progress: true
+
 jobs:
   test:
     runs-on: ubuntu-18.04

.github/workflows/ui-tests.yml

@@ -6,6 +6,10 @@ on:
   push:
     branches: [ develop ]
 
+concurrency:
+  group: ui-develop-${{ github.event.number }}
+  cancel-in-progress: true
+
 jobs:
   test:
     runs-on: ubuntu-18.04

CODEOWNERS

@@ -7,10 +7,13 @@
 templates/				@surajshetty3416
 www/							@surajshetty3416
 integrations/			@leela
-patches/					@surajshetty3416
+patches/					@surajshetty3416 @gavindsouza
 email/						@leela
 event_streaming/	@ruchamahabal
 data_import*			@netchampfaris
 core/							@surajshetty3416
+database          @gavindsouza
+model             @gavindsouza
 requirements.txt	@gavindsouza
 commands/					@gavindsouza
+workspace         @shariquerik

cypress/integration/datetime_field_form_validation.js

@@ -0,0 +1,19 @@
+context('Datetime Field Validation', () => {
+	before(() => {
+		cy.login();
+		cy.visit('/app/communication');
+		cy.window().its('frappe').then(frappe => {
+			frappe.call("frappe.tests.ui_test_helpers.create_communication_records");
+		});
+	});
+
+	// validating datetime field value when value is set from backend and get validated on form load.
+	it('datetime field form validation', () => {
+		cy.visit('/app/communication');
+		cy.get('a[title="Test Form Communication 1"]').invoke('attr', 'data-name')
+			.then((name) => {
+				cy.visit(`/app/communication/${name}`);
+				cy.get('.indicator-pill').should('contain', 'Open').should('have.class', 'red');
+			});
+	});
+});

cypress/integration/folder_navigation.js

@@ -0,0 +1,79 @@
+context('Folder Navigation', () => {
+	before(() => {
+		cy.visit('/login');
+		cy.login();
+		cy.visit('/app/file');
+	});
+
+	it('Adding Folders', () => {
+		//Adding filter to go into the home folder
+		cy.get('.filter-selector > .btn').findByText('1 filter').click();
+		cy.findByRole('button', {name: 'Clear Filters'}).click();
+		cy.get('.filter-action-buttons > .text-muted').findByText('+ Add a Filter').click();
+		cy.get('.fieldname-select-area > .awesomplete > .form-control').type('Fol{enter}');
+		cy.get('.filter-field > .form-group > .link-field > .awesomplete > .input-with-feedback').type('Home{enter}');
+		cy.get('.filter-action-buttons > div > .btn-primary').findByText('Apply Filters').click();
+
+		//Adding folder (Test Folder)
+		cy.get('.menu-btn-group > .btn').click();
+		cy.get('.menu-btn-group [data-label="New Folder"]').click();
+		cy.get('form > [data-fieldname="value"]').type('Test Folder');
+		cy.findByRole('button', {name: 'Create'}).click();   
+	});
+
+	it('Navigating the nested folders, checking if the URL formed is correct, checking if the added content in the child folder is correct', () => {
+		//Navigating inside the Attachments folder
+		cy.get('[title="Attachments"] > span').click();
+
+		//To check if the URL formed after visiting the attachments folder is correct
+		cy.location('pathname').should('eq', '/app/file/view/home/Attachments');
+		cy.visit('/app/file/view/home/Attachments');
+
+		//Adding folder inside the attachments folder
+		cy.get('.menu-btn-group > .btn').click();
+		cy.get('.menu-btn-group [data-label="New Folder"]').click();
+		cy.get('form > [data-fieldname="value"]').type('Test Folder');
+		cy.findByRole('button', {name: 'Create'}).click();
+
+		//Navigating inside the added folder in the Attachments folder
+		cy.get('[title="Test Folder"] > span').click();
+
+		//To check if the URL is correct after visiting the Test Folder
+		cy.location('pathname').should('eq', '/app/file/view/home/Attachments/Test%20Folder');
+		cy.visit('/app/file/view/home/Attachments/Test%20Folder');
+
+		//Adding a file inside the Test Folder
+		cy.findByRole('button', {name: 'Add File'}).eq(0).click({force: true});
+		cy.get('.file-uploader').findByText('Link').click();
+		cy.get('.input-group > .form-control').type('https://wallpaperplay.com/walls/full/8/2/b/72402.jpg');
+		cy.findByRole('button', {name: 'Upload'}).click();
+
+		//To check if the added file is present in the Test Folder
+		cy.get('span.level-item > span').should('contain', 'Test Folder');
+		cy.get('.list-row-container').eq(0).should('contain.text', '72402.jpg');
+		cy.get('.list-row-checkbox').eq(0).click();
+
+		//Deleting the added file from the Test folder
+		cy.findByRole('button', {name: 'Actions'}).click();
+		cy.get('.actions-btn-group [data-label="Delete"]').click();
+		cy.wait(700);
+		cy.findByRole('button', {name: 'Yes'}).click();
+		cy.wait(700);
+
+		//Deleting the Test Folder
+		cy.visit('/app/file/view/home/Attachments');
+		cy.get('.list-row-checkbox').eq(0).click();
+		cy.findByRole('button', {name: 'Actions'}).click();
+		cy.get('.actions-btn-group [data-label="Delete"]').click();  
+		cy.findByRole('button', {name: 'Yes'}).click();
+	});
+
+	it('Deleting Test Folder from the home', () => {
+	//Deleting the Test Folder added in the home directory
+		cy.visit('/app/file/view/home');
+		cy.get('.level-left > .list-subject > .list-row-checkbox').eq(0).click({force: true, delay: 500});
+		cy.findByRole('button', {name: 'Actions'}).click();
+		cy.get('.actions-btn-group [data-label="Delete"]').click();
+		cy.findByRole('button', {name: 'Yes'}).click();
+	});
+}); 

frappe/core/doctype/file/file.js

@@ -29,15 +29,8 @@ frappe.ui.form.on("File", "refresh", function(frm) {
 	if (is_optimizable) {
 		frm.add_custom_button(__("Optimize"), function() {
 			frappe.show_alert(__("Optimizing image..."));
-			frappe.call({
-				method: "frappe.core.doctype.file.file.optimize_saved_image",
-				args: {
-					doc_name: frm.doc.name,
-				},
-				callback: function() {
-					frappe.show_alert(__("Image optimized"));
-					frappe.set_route("List", "File");
-				}
+			frm.call("optimize_file").then(() => {
+				frappe.show_alert(__("Image optimized"));
 			});
 		});
 	}

frappe/core/doctype/file/file.py

@@ -313,8 +313,16 @@ def call_delete_file(self):
 			self.delete_file_data_content(only_thumbnail=True)
 
 	def on_rollback(self):
-		self.flags.on_rollback = True
-		self.on_trash()
+		# if original_content flag is set, this rollback should revert the file to its original state
+		if self.flags.original_content:
+			file_path = self.get_full_path()
+			with open(file_path, "wb+") as f:
+				f.write(self.flags.original_content)
+
+		# following condition is only executed when an insert has been rolledback
+		else:
+			self.flags.on_rollback = True
+			self.on_trash()
 
 	def unzip(self):
 		'''Unzip current file and replace it by its children'''
@@ -531,6 +539,35 @@ def set_is_private(self):
 		if self.file_url:
 			self.is_private = cint(self.file_url.startswith('/private'))
 
+	@frappe.whitelist()
+	def optimize_file(self):
+		if self.is_folder:
+			raise TypeError('Folders cannot be optimized')
+
+		content_type = mimetypes.guess_type(self.file_name)[0]
+		is_local_image = content_type.startswith('image/') and self.file_size > 0
+		is_svg = content_type == 'image/svg+xml'
+
+		if not is_local_image:
+			raise NotImplementedError('Only local image files can be optimized')
+
+		if is_svg:
+			raise TypeError('Optimization of SVG images is not supported')
+
+		content = self.get_content()
+		file_path = self.get_full_path()
+		optimized_content = optimize_image(content, content_type)
+
+		with open(file_path, 'wb+') as f:
+			f.write(optimized_content)
+
+		self.file_size = len(optimized_content)
+		self.content_hash = get_content_hash(optimized_content)
+		# if rolledback, revert back to original
+		self.flags.original_content = content
+		frappe.local.rollback_observers.append(self)
+		self.save()
+
 def on_doctype_update():
 	frappe.db.add_index("File", ["attached_to_doctype", "attached_to_name"])
 
@@ -838,22 +875,6 @@ def unzip_file(name):
 	files = file_obj.unzip()
 	return files
 
-@frappe.whitelist()
-def optimize_saved_image(doc_name):
-	file_doc = frappe.get_doc('File', doc_name)
-	content = file_doc.get_content()
-	content_type = mimetypes.guess_type(file_doc.file_name)[0]
-
-	optimized_content = optimize_image(content, content_type)
-
-	file_path = get_files_path(is_private=file_doc.is_private)
-	file_path = os.path.join(file_path.encode('utf-8'), file_doc.file_name.encode('utf-8'))
-	with open(file_path, 'wb+') as f:
-		f.write(optimized_content)
-
-	file_doc.file_size = len(optimized_content)
-	file_doc.content_hash = get_content_hash(optimized_content)
-	file_doc.save()
 
 @frappe.whitelist()
 def get_attached_images(doctype, names):

frappe/core/doctype/file/test_file.py

@@ -440,6 +440,7 @@ def test_file_unzip(self):
 		}).insert(ignore_permissions=True)
 		self.assertRaisesRegex(frappe.exceptions.ValidationError, 'not a zip file', test_file.unzip)
 
+
 class TestAttachment(unittest.TestCase):
 	test_doctype = 'Test For Attachment'
 
@@ -569,3 +570,68 @@ def test_create_new_folder(self):
 		from frappe.core.doctype.file.file import create_new_folder
 		folder = create_new_folder('test_folder', 'Home')
 		self.assertTrue(folder.is_folder)
+
+
+class TestFileOptimization(unittest.TestCase):
+	def test_optimize_file(self):
+		file_path = frappe.get_app_path("frappe", "tests/data/sample_image_for_optimization.jpg")
+		with open(file_path, "rb") as f:
+			file_content = f.read()
+		test_file = frappe.get_doc({
+			"doctype": "File",
+			"file_name": "sample_image_for_optimization.jpg",
+			"content": file_content
+		}).insert()
+		original_size = test_file.file_size
+		original_content_hash = test_file.content_hash
+
+		test_file.optimize_file()
+		optimized_size = test_file.file_size
+		updated_content_hash = test_file.content_hash
+
+		self.assertLess(optimized_size, original_size)
+		self.assertNotEqual(original_content_hash, updated_content_hash)
+		test_file.delete()
+
+	def test_optimize_svg(self):
+		file_path = frappe.get_app_path("frappe", "tests/data/sample_svg.svg")
+		with open(file_path, "rb") as f:
+			file_content = f.read()
+		test_file = frappe.get_doc({
+			"doctype": "File",
+			"file_name": "sample_svg.svg",
+			"content": file_content
+		}).insert()
+		self.assertRaises(TypeError, test_file.optimize_file)
+		test_file.delete()
+
+	def test_optimize_textfile(self):
+		test_file = frappe.get_doc({
+			"doctype": "File",
+			"file_name": "sample_text.txt",
+			"content": "Text files cannot be optimized"
+		}).insert()
+		self.assertRaises(NotImplementedError, test_file.optimize_file)
+		test_file.delete()
+
+	def test_optimize_folder(self):
+		test_folder = frappe.get_doc("File", "Home/Attachments")
+		self.assertRaises(TypeError, test_folder.optimize_file)
+
+	def test_revert_optimized_file_on_rollback(self):
+		file_path = frappe.get_app_path("frappe", "tests/data/sample_image_for_optimization.jpg")
+		with open(file_path, "rb") as f:
+			file_content = f.read()
+		test_file = frappe.get_doc({
+			"doctype": "File",
+			"file_name": "sample_image_for_optimization.jpg",
+			"content": file_content
+		}).insert()
+		image_path = test_file.get_full_path()
+		size_before_optimization = os.stat(image_path).st_size
+
+		test_file.optimize_file()
+		frappe.db.rollback()
+		size_after_rollback = os.stat(image_path).st_size
+		self.assertEqual(size_before_optimization, size_after_rollback)
+		test_file.delete()